2019-02-18 12:56:30 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #413 daemon prio=9 os_prio=31 tid=0x00007fd8bc8d8000 nid=0x1450b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@359f04ca" #412 daemon prio=5 os_prio=31 tid=0x00007fd8ba4e7000 nid=0x1360b waiting on condition [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #411 daemon prio=5 os_prio=31 tid=0x00007fd8c00e9800 nid=0x13f0b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #410 daemon prio=5 os_prio=31 tid=0x00007fd8bf868800 nid=0x1340b waiting on condition [0x00007000085cd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #409 daemon prio=5 os_prio=31 tid=0x00007fd8c00e8800 nid=0x1491b waiting on condition [0x00007000084ca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007990d02e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007fd8c009a000 nid=0x11813 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007fd8c0099800 nid=0x15707 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007fd8bf9fa000 nid=0x15507 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #403 daemon prio=5 os_prio=31 tid=0x00007fd8be875800 nid=0x15307 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #402 daemon prio=5 os_prio=31 tid=0x00007fd8bf862000 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #400 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1800 nid=0x1510f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796a4fd60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #399 daemon prio=5 os_prio=31 tid=0x00007fd8be83a000 nid=0x11213 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796e4c318> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #398 daemon prio=5 os_prio=31 tid=0x00007fd8c00d2800 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000797054e48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #397 daemon prio=5 os_prio=31 tid=0x00007fd8ba320800 nid=0x991f in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000797032d30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #395 daemon prio=5 os_prio=31 tid=0x00007fd8ba294800 nid=0xfb0f in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e83690> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #394 daemon prio=5 os_prio=31 tid=0x00007fd8be813000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e57e00> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #393 daemon prio=5 os_prio=31 tid=0x00007fd8bea27000 nid=0x11d13 in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e55428> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #392 daemon prio=5 os_prio=31 tid=0x00007fd8bea1b000 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e52c70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #391 daemon prio=5 os_prio=31 tid=0x00007fd8bf947800 nid=0xfc1b runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079701a1e8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796e50e00> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x0000000796a53078> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #377 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1000 nid=0xee17 in Object.wait() [0x0000700008de5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000795ac4a40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #374 daemon prio=5 os_prio=31 tid=0x00007fd8bf8a8800 nid=0xf20f in Object.wait() [0x0000700008bdf000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007958f3248> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #353 daemon prio=5 os_prio=31 tid=0x00007fd8c00e0000 nid=0x14107 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a098b480> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007fd8ba42d000 nid=0x13703 in Object.wait() [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a096b618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cf000 nid=0x11f0b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a09f81c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4da8a050" #306 daemon prio=5 os_prio=31 tid=0x00007fd8be812800 nid=0x10f07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007fd8ba436800 nid=0x10d07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007fd8ba436000 nid=0x10b07 waiting on condition [0x0000700007fbb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007fd8ba435000 nid=0xf70f waiting on condition [0x0000700007eb8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0aa6d50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007fd8bf962000 nid=0x1090f waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007fd8ba2ae000 nid=0x10707 waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007fd8c019d800 nid=0x10507 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007fd8c019d000 nid=0x10307 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007fd8c019c000 nid=0x10107 waiting on condition [0x00007000079a9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007fd8bf961800 nid=0xff07 waiting on condition [0x00007000078a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09f9110> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007fd8be811800 nid=0x3e0b in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aaa9d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007fd8c019a800 nid=0x6107 in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007fd8be811000 nid=0x5d07 in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acd940> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007fd8be810000 nid=0x7b17 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab628> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007fd8be83b000 nid=0xec0b in Object.wait() [0x0000700000e3d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aabe58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007fd8bf861000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0abf2d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007fd8bf865800 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aceba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f9000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acf3a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1f800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0891c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4323fbfd" #221 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1e000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007fd8be80d800 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007fd8ba291000 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007fd8ba290800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0a902b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007fd8bc1b1800 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007fd8bea57800 nid=0xde03 waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1d000 nid=0x880f waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007fd8bea56800 nid=0x3c0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007fd8bc8c4000 nid=0x8a0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f3800 nid=0x8d0f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0892de8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007fd8ba69f800 nid=0x840f in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007fd8bbc71000 nid=0xa01b in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2dad8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007fd8bea59000 nid=0x7e0f in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2f278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007fd8ba5ed000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007fd8be9f3800 nid=0xaa0b in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007fd8bbb75800 nid=0x760f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740ae72c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@5ab03154" #198 daemon prio=5 os_prio=31 tid=0x00007fd8ba2b0000 nid=0x690f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007fd8ba2af000 nid=0x670f waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007fd8bf98f000 nid=0x7217 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007fd8bf98e800 nid=0xc80f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a08d19d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007fd8ba279800 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007fd8bbc5a000 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007fd8ba279000 nid=0xd807 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007fd8c0055000 nid=0xc407 waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007fd8bf8bd000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007fd8ba351000 nid=0xd40f waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09056d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007fd8ba218800 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740a8fd70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007fd8ba218000 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007fd8bbcae800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91130> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007fd8bbdb5000 nid=0xca07 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a917f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007fd8ba33c000 nid=0xc007 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007fd8ba25b000 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007fd8ba136800 nid=0xbc07 in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007fd8bea58000 nid=0xc203 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740accbf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@76053569" #151 daemon prio=5 os_prio=31 tid=0x00007fd8ba25a000 nid=0xba03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f3800 nid=0xb803 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007fd8bea56000 nid=0xb603 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007fd8bea54800 nid=0xb40b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acd3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007fd8bfa7f000 nid=0xb20b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740b2e648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007fd8bbbcb800 nid=0xb003 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007fd8be809800 nid=0x6217 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f2800 nid=0x9e0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007fd8bbde1800 nid=0x660b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007fd8bbb57000 nid=0x740b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007fd8bbb7b800 nid=0x960b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acf0a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c6800 nid=0xa813 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc3ed0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c5800 nid=0xa60b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be3f68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007fd8bea54000 nid=0xa40b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b800 nid=0x920b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b000 nid=0xa213 in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007fd8ba1b5800 nid=0x6e0b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b2dd50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cc800 nid=0x8213 in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be5548> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007fd8bf8c8800 nid=0x7807 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cb9400> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007fd8baaea000 nid=0x8b07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cc5dc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007fd8bac06800 nid=0x9003 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740aa5078> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007fd8ba361000 nid=0x8003 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740b088f0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007fd8baa09000 nid=0x6a03 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000074001bf40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #12 daemon prio=10 os_prio=31 tid=0x00007fd8ba9c9800 nid=0x5b03 waiting on condition [0x0000700001786000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c440> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007fd8b9904000 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007400375a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007fd8b98f8800 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007400317a0> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007fd8bb80d000 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007fd8ba028800 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007fd8ba822800 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007fd8bb800800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007fd8b985e800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007fd8b9840800 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007fd8b984c000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007fd8b980c800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007fd8bb802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007fd8ba80a800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813000 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813800 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007fd8b9814000 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007fd8bb80a800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007fd8ba002000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003000 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003800 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007fd8ba004000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007fd8bb801000 nid=0x5503 waiting on condition JNI global references: 301 2019-02-18 12:56:34 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #413 daemon prio=9 os_prio=31 tid=0x00007fd8bc8d8000 nid=0x1450b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@359f04ca" #412 daemon prio=5 os_prio=31 tid=0x00007fd8ba4e7000 nid=0x1360b waiting on condition [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #411 daemon prio=5 os_prio=31 tid=0x00007fd8c00e9800 nid=0x13f0b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #410 daemon prio=5 os_prio=31 tid=0x00007fd8bf868800 nid=0x1340b waiting on condition [0x00007000085cd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #409 daemon prio=5 os_prio=31 tid=0x00007fd8c00e8800 nid=0x1491b waiting on condition [0x00007000084ca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007990d02e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007fd8c009a000 nid=0x11813 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007fd8c0099800 nid=0x15707 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007fd8bf9fa000 nid=0x15507 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #403 daemon prio=5 os_prio=31 tid=0x00007fd8be875800 nid=0x15307 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #402 daemon prio=5 os_prio=31 tid=0x00007fd8bf862000 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #400 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1800 nid=0x1510f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796a4fd60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #399 daemon prio=5 os_prio=31 tid=0x00007fd8be83a000 nid=0x11213 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796e4c318> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #398 daemon prio=5 os_prio=31 tid=0x00007fd8c00d2800 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000797054e48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #397 daemon prio=5 os_prio=31 tid=0x00007fd8ba320800 nid=0x991f in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000797032d30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #395 daemon prio=5 os_prio=31 tid=0x00007fd8ba294800 nid=0xfb0f in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e83690> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #394 daemon prio=5 os_prio=31 tid=0x00007fd8be813000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e57e00> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #393 daemon prio=5 os_prio=31 tid=0x00007fd8bea27000 nid=0x11d13 in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e55428> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #392 daemon prio=5 os_prio=31 tid=0x00007fd8bea1b000 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e52c70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #391 daemon prio=5 os_prio=31 tid=0x00007fd8bf947800 nid=0xfc1b runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079701a1e8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796e50e00> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x0000000796a53078> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #377 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1000 nid=0xee17 in Object.wait() [0x0000700008de5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000795ac4a40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #374 daemon prio=5 os_prio=31 tid=0x00007fd8bf8a8800 nid=0xf20f in Object.wait() [0x0000700008bdf000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007958f3248> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #353 daemon prio=5 os_prio=31 tid=0x00007fd8c00e0000 nid=0x14107 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a098b480> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007fd8ba42d000 nid=0x13703 in Object.wait() [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a096b618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cf000 nid=0x11f0b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a09f81c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4da8a050" #306 daemon prio=5 os_prio=31 tid=0x00007fd8be812800 nid=0x10f07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007fd8ba436800 nid=0x10d07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007fd8ba436000 nid=0x10b07 waiting on condition [0x0000700007fbb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007fd8ba435000 nid=0xf70f waiting on condition [0x0000700007eb8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0aa6d50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007fd8bf962000 nid=0x1090f waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007fd8ba2ae000 nid=0x10707 waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007fd8c019d800 nid=0x10507 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007fd8c019d000 nid=0x10307 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007fd8c019c000 nid=0x10107 waiting on condition [0x00007000079a9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007fd8bf961800 nid=0xff07 waiting on condition [0x00007000078a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09f9110> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007fd8be811800 nid=0x3e0b in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aaa9d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007fd8c019a800 nid=0x6107 in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007fd8be811000 nid=0x5d07 in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acd940> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007fd8be810000 nid=0x7b17 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab628> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007fd8be83b000 nid=0xec0b in Object.wait() [0x0000700000e3d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aabe58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007fd8bf861000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0abf2d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007fd8bf865800 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aceba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f9000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acf3a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1f800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0891c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4323fbfd" #221 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1e000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007fd8be80d800 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007fd8ba291000 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007fd8ba290800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0a902b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007fd8bc1b1800 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007fd8bea57800 nid=0xde03 waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1d000 nid=0x880f waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007fd8bea56800 nid=0x3c0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007fd8bc8c4000 nid=0x8a0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f3800 nid=0x8d0f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0892de8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007fd8ba69f800 nid=0x840f in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007fd8bbc71000 nid=0xa01b in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2dad8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007fd8bea59000 nid=0x7e0f in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2f278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007fd8ba5ed000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007fd8be9f3800 nid=0xaa0b in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007fd8bbb75800 nid=0x760f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740ae72c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@5ab03154" #198 daemon prio=5 os_prio=31 tid=0x00007fd8ba2b0000 nid=0x690f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007fd8ba2af000 nid=0x670f waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007fd8bf98f000 nid=0x7217 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007fd8bf98e800 nid=0xc80f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a08d19d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007fd8ba279800 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007fd8bbc5a000 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007fd8ba279000 nid=0xd807 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007fd8c0055000 nid=0xc407 waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007fd8bf8bd000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007fd8ba351000 nid=0xd40f waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09056d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007fd8ba218800 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740a8fd70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007fd8ba218000 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007fd8bbcae800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91130> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007fd8bbdb5000 nid=0xca07 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a917f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007fd8ba33c000 nid=0xc007 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007fd8ba25b000 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007fd8ba136800 nid=0xbc07 in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007fd8bea58000 nid=0xc203 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740accbf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@76053569" #151 daemon prio=5 os_prio=31 tid=0x00007fd8ba25a000 nid=0xba03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f3800 nid=0xb803 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007fd8bea56000 nid=0xb603 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007fd8bea54800 nid=0xb40b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acd3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007fd8bfa7f000 nid=0xb20b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740b2e648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007fd8bbbcb800 nid=0xb003 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007fd8be809800 nid=0x6217 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f2800 nid=0x9e0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007fd8bbde1800 nid=0x660b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007fd8bbb57000 nid=0x740b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007fd8bbb7b800 nid=0x960b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acf0a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c6800 nid=0xa813 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc3ed0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c5800 nid=0xa60b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be3f68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007fd8bea54000 nid=0xa40b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b800 nid=0x920b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b000 nid=0xa213 in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007fd8ba1b5800 nid=0x6e0b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b2dd50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cc800 nid=0x8213 in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be5548> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007fd8bf8c8800 nid=0x7807 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cb9400> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007fd8baaea000 nid=0x8b07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cc5dc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007fd8bac06800 nid=0x9003 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740aa5078> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007fd8ba361000 nid=0x8003 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740b088f0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007fd8baa09000 nid=0x6a03 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000074001bf40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #12 daemon prio=10 os_prio=31 tid=0x00007fd8ba9c9800 nid=0x5b03 waiting on condition [0x0000700001786000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c440> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007fd8b9904000 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007400375a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007fd8b98f8800 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007400317a0> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007fd8bb80d000 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007fd8ba028800 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007fd8ba822800 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007fd8bb800800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007fd8b985e800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007fd8b9840800 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007fd8b984c000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007fd8b980c800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007fd8bb802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007fd8ba80a800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813000 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813800 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007fd8b9814000 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007fd8bb80a800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007fd8ba002000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003000 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003800 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007fd8ba004000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007fd8bb801000 nid=0x5503 waiting on condition JNI global references: 301 2019-02-18 12:56:36 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #413 daemon prio=9 os_prio=31 tid=0x00007fd8bc8d8000 nid=0x1450b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@359f04ca" #412 daemon prio=5 os_prio=31 tid=0x00007fd8ba4e7000 nid=0x1360b waiting on condition [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #411 daemon prio=5 os_prio=31 tid=0x00007fd8c00e9800 nid=0x13f0b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #410 daemon prio=5 os_prio=31 tid=0x00007fd8bf868800 nid=0x1340b waiting on condition [0x00007000085cd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #409 daemon prio=5 os_prio=31 tid=0x00007fd8c00e8800 nid=0x1491b waiting on condition [0x00007000084ca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007990d02e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007fd8c009a000 nid=0x11813 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007fd8c0099800 nid=0x15707 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007fd8bf9fa000 nid=0x15507 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #403 daemon prio=5 os_prio=31 tid=0x00007fd8be875800 nid=0x15307 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #402 daemon prio=5 os_prio=31 tid=0x00007fd8bf862000 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #400 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1800 nid=0x1510f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796a4fd60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #399 daemon prio=5 os_prio=31 tid=0x00007fd8be83a000 nid=0x11213 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796e4c318> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #398 daemon prio=5 os_prio=31 tid=0x00007fd8c00d2800 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000797054e48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #397 daemon prio=5 os_prio=31 tid=0x00007fd8ba320800 nid=0x991f in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000797032d30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #395 daemon prio=5 os_prio=31 tid=0x00007fd8ba294800 nid=0xfb0f in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e83690> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #394 daemon prio=5 os_prio=31 tid=0x00007fd8be813000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e57e00> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #393 daemon prio=5 os_prio=31 tid=0x00007fd8bea27000 nid=0x11d13 in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e55428> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #392 daemon prio=5 os_prio=31 tid=0x00007fd8bea1b000 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e52c70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #391 daemon prio=5 os_prio=31 tid=0x00007fd8bf947800 nid=0xfc1b runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079701a1e8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796e50e00> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x0000000796a53078> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #377 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1000 nid=0xee17 in Object.wait() [0x0000700008de5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000795ac4a40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #374 daemon prio=5 os_prio=31 tid=0x00007fd8bf8a8800 nid=0xf20f in Object.wait() [0x0000700008bdf000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007958f3248> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #353 daemon prio=5 os_prio=31 tid=0x00007fd8c00e0000 nid=0x14107 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a098b480> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007fd8ba42d000 nid=0x13703 in Object.wait() [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a096b618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cf000 nid=0x11f0b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a09f81c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4da8a050" #306 daemon prio=5 os_prio=31 tid=0x00007fd8be812800 nid=0x10f07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007fd8ba436800 nid=0x10d07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007fd8ba436000 nid=0x10b07 waiting on condition [0x0000700007fbb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007fd8ba435000 nid=0xf70f waiting on condition [0x0000700007eb8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0aa6d50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007fd8bf962000 nid=0x1090f waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007fd8ba2ae000 nid=0x10707 waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007fd8c019d800 nid=0x10507 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007fd8c019d000 nid=0x10307 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007fd8c019c000 nid=0x10107 waiting on condition [0x00007000079a9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007fd8bf961800 nid=0xff07 waiting on condition [0x00007000078a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09f9110> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007fd8be811800 nid=0x3e0b in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aaa9d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007fd8c019a800 nid=0x6107 in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007fd8be811000 nid=0x5d07 in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acd940> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007fd8be810000 nid=0x7b17 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab628> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007fd8be83b000 nid=0xec0b in Object.wait() [0x0000700000e3d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aabe58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007fd8bf861000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0abf2d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007fd8bf865800 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aceba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f9000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acf3a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1f800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0891c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4323fbfd" #221 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1e000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007fd8be80d800 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007fd8ba291000 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007fd8ba290800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0a902b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007fd8bc1b1800 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007fd8bea57800 nid=0xde03 waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1d000 nid=0x880f waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007fd8bea56800 nid=0x3c0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007fd8bc8c4000 nid=0x8a0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f3800 nid=0x8d0f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0892de8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007fd8ba69f800 nid=0x840f in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007fd8bbc71000 nid=0xa01b in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2dad8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007fd8bea59000 nid=0x7e0f in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2f278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007fd8ba5ed000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007fd8be9f3800 nid=0xaa0b in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007fd8bbb75800 nid=0x760f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740ae72c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@5ab03154" #198 daemon prio=5 os_prio=31 tid=0x00007fd8ba2b0000 nid=0x690f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007fd8ba2af000 nid=0x670f waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007fd8bf98f000 nid=0x7217 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007fd8bf98e800 nid=0xc80f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a08d19d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007fd8ba279800 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007fd8bbc5a000 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007fd8ba279000 nid=0xd807 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007fd8c0055000 nid=0xc407 waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007fd8bf8bd000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007fd8ba351000 nid=0xd40f waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09056d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007fd8ba218800 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740a8fd70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007fd8ba218000 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007fd8bbcae800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91130> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007fd8bbdb5000 nid=0xca07 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a917f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007fd8ba33c000 nid=0xc007 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007fd8ba25b000 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007fd8ba136800 nid=0xbc07 in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007fd8bea58000 nid=0xc203 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740accbf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@76053569" #151 daemon prio=5 os_prio=31 tid=0x00007fd8ba25a000 nid=0xba03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f3800 nid=0xb803 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007fd8bea56000 nid=0xb603 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007fd8bea54800 nid=0xb40b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acd3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007fd8bfa7f000 nid=0xb20b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740b2e648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007fd8bbbcb800 nid=0xb003 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007fd8be809800 nid=0x6217 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f2800 nid=0x9e0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007fd8bbde1800 nid=0x660b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007fd8bbb57000 nid=0x740b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007fd8bbb7b800 nid=0x960b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acf0a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c6800 nid=0xa813 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc3ed0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c5800 nid=0xa60b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be3f68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007fd8bea54000 nid=0xa40b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b800 nid=0x920b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b000 nid=0xa213 in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007fd8ba1b5800 nid=0x6e0b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b2dd50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cc800 nid=0x8213 in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be5548> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007fd8bf8c8800 nid=0x7807 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cb9400> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007fd8baaea000 nid=0x8b07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cc5dc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007fd8bac06800 nid=0x9003 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740aa5078> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007fd8ba361000 nid=0x8003 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740b088f0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007fd8baa09000 nid=0x6a03 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000074001bf40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #12 daemon prio=10 os_prio=31 tid=0x00007fd8ba9c9800 nid=0x5b03 waiting on condition [0x0000700001786000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c440> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007fd8b9904000 nid=0x5903 runnable [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007400375a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007fd8b98f8800 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007400317a0> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007fd8bb80d000 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007fd8ba028800 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007fd8ba822800 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007fd8bb800800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007fd8b985e800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007fd8b9840800 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007fd8b984c000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007fd8b980c800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007fd8bb802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007fd8ba80a800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813000 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813800 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007fd8b9814000 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007fd8bb80a800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007fd8ba002000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003000 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003800 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007fd8ba004000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007fd8bb801000 nid=0x5503 waiting on condition JNI global references: 301 2019-02-18 12:59:10 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #413 daemon prio=9 os_prio=31 tid=0x00007fd8bc8d8000 nid=0x1450b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@359f04ca" #412 daemon prio=5 os_prio=31 tid=0x00007fd8ba4e7000 nid=0x1360b waiting on condition [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #411 daemon prio=5 os_prio=31 tid=0x00007fd8c00e9800 nid=0x13f0b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #410 daemon prio=5 os_prio=31 tid=0x00007fd8bf868800 nid=0x1340b waiting on condition [0x00007000085cd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #409 daemon prio=5 os_prio=31 tid=0x00007fd8c00e8800 nid=0x1491b waiting on condition [0x00007000084ca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007990d02e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007fd8c009a000 nid=0x11813 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007fd8c0099800 nid=0x15707 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007fd8bf9fa000 nid=0x15507 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #403 daemon prio=5 os_prio=31 tid=0x00007fd8be875800 nid=0x15307 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #402 daemon prio=5 os_prio=31 tid=0x00007fd8bf862000 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #400 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1800 nid=0x1510f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796a4fd60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #399 daemon prio=5 os_prio=31 tid=0x00007fd8be83a000 nid=0x11213 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796e4c318> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #398 daemon prio=5 os_prio=31 tid=0x00007fd8c00d2800 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000797054e48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #397 daemon prio=5 os_prio=31 tid=0x00007fd8ba320800 nid=0x991f in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000797032d30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #395 daemon prio=5 os_prio=31 tid=0x00007fd8ba294800 nid=0xfb0f in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e83690> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #394 daemon prio=5 os_prio=31 tid=0x00007fd8be813000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e57e00> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #393 daemon prio=5 os_prio=31 tid=0x00007fd8bea27000 nid=0x11d13 in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e55428> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #392 daemon prio=5 os_prio=31 tid=0x00007fd8bea1b000 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e52c70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #391 daemon prio=5 os_prio=31 tid=0x00007fd8bf947800 nid=0xfc1b runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079701a1e8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796e50e00> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x0000000796a53078> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #377 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1000 nid=0xee17 in Object.wait() [0x0000700008de5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000795ac4a40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #374 daemon prio=5 os_prio=31 tid=0x00007fd8bf8a8800 nid=0xf20f in Object.wait() [0x0000700008bdf000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007958f3248> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #353 daemon prio=5 os_prio=31 tid=0x00007fd8c00e0000 nid=0x14107 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a098b480> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007fd8ba42d000 nid=0x13703 in Object.wait() [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a096b618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cf000 nid=0x11f0b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a09f81c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4da8a050" #306 daemon prio=5 os_prio=31 tid=0x00007fd8be812800 nid=0x10f07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007fd8ba436800 nid=0x10d07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007fd8ba436000 nid=0x10b07 waiting on condition [0x0000700007fbb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007fd8ba435000 nid=0xf70f waiting on condition [0x0000700007eb8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0aa6d50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007fd8bf962000 nid=0x1090f waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007fd8ba2ae000 nid=0x10707 waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007fd8c019d800 nid=0x10507 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007fd8c019d000 nid=0x10307 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007fd8c019c000 nid=0x10107 waiting on condition [0x00007000079a9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007fd8bf961800 nid=0xff07 waiting on condition [0x00007000078a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09f9110> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007fd8be811800 nid=0x3e0b in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aaa9d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007fd8c019a800 nid=0x6107 in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007fd8be811000 nid=0x5d07 in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acd940> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007fd8be810000 nid=0x7b17 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab628> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007fd8be83b000 nid=0xec0b in Object.wait() [0x0000700000e3d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aabe58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007fd8bf861000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0abf2d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007fd8bf865800 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aceba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f9000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acf3a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1f800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0891c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4323fbfd" #221 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1e000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007fd8be80d800 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007fd8ba291000 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007fd8ba290800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0a902b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007fd8bc1b1800 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007fd8bea57800 nid=0xde03 waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1d000 nid=0x880f waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007fd8bea56800 nid=0x3c0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007fd8bc8c4000 nid=0x8a0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f3800 nid=0x8d0f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0892de8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007fd8ba69f800 nid=0x840f in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007fd8bbc71000 nid=0xa01b in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2dad8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007fd8bea59000 nid=0x7e0f in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2f278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007fd8ba5ed000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007fd8be9f3800 nid=0xaa0b in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007fd8bbb75800 nid=0x760f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740ae72c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@5ab03154" #198 daemon prio=5 os_prio=31 tid=0x00007fd8ba2b0000 nid=0x690f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007fd8ba2af000 nid=0x670f waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007fd8bf98f000 nid=0x7217 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007fd8bf98e800 nid=0xc80f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a08d19d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007fd8ba279800 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007fd8bbc5a000 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007fd8ba279000 nid=0xd807 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007fd8c0055000 nid=0xc407 waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007fd8bf8bd000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007fd8ba351000 nid=0xd40f waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09056d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007fd8ba218800 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740a8fd70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007fd8ba218000 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007fd8bbcae800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91130> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007fd8bbdb5000 nid=0xca07 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a917f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007fd8ba33c000 nid=0xc007 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007fd8ba25b000 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007fd8ba136800 nid=0xbc07 in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007fd8bea58000 nid=0xc203 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740accbf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@76053569" #151 daemon prio=5 os_prio=31 tid=0x00007fd8ba25a000 nid=0xba03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f3800 nid=0xb803 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007fd8bea56000 nid=0xb603 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007fd8bea54800 nid=0xb40b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acd3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007fd8bfa7f000 nid=0xb20b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740b2e648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007fd8bbbcb800 nid=0xb003 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007fd8be809800 nid=0x6217 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f2800 nid=0x9e0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007fd8bbde1800 nid=0x660b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007fd8bbb57000 nid=0x740b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007fd8bbb7b800 nid=0x960b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acf0a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c6800 nid=0xa813 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc3ed0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c5800 nid=0xa60b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be3f68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007fd8bea54000 nid=0xa40b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b800 nid=0x920b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b000 nid=0xa213 in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007fd8ba1b5800 nid=0x6e0b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b2dd50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cc800 nid=0x8213 in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be5548> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007fd8bf8c8800 nid=0x7807 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cb9400> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007fd8baaea000 nid=0x8b07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cc5dc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007fd8bac06800 nid=0x9003 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740aa5078> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007fd8ba361000 nid=0x8003 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740b088f0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007fd8baa09000 nid=0x6a03 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000074001bf40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #12 daemon prio=10 os_prio=31 tid=0x00007fd8ba9c9800 nid=0x5b03 waiting on condition [0x0000700001786000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c440> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007fd8b9904000 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007400375a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007fd8b98f8800 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007400317a0> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007fd8bb80d000 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007fd8ba028800 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007fd8ba822800 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007fd8bb800800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007fd8b985e800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007fd8b9840800 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007fd8b984c000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007fd8b980c800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007fd8bb802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007fd8ba80a800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813000 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813800 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007fd8b9814000 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007fd8bb80a800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007fd8ba002000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003000 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003800 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007fd8ba004000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007fd8bb801000 nid=0x5503 waiting on condition JNI global references: 301 2019-02-18 13:00:54 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #413 daemon prio=9 os_prio=31 tid=0x00007fd8bc8d8000 nid=0x1450b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@359f04ca" #412 daemon prio=5 os_prio=31 tid=0x00007fd8ba4e7000 nid=0x1360b waiting on condition [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #411 daemon prio=5 os_prio=31 tid=0x00007fd8c00e9800 nid=0x13f0b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #410 daemon prio=5 os_prio=31 tid=0x00007fd8bf868800 nid=0x1340b waiting on condition [0x00007000085cd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #409 daemon prio=5 os_prio=31 tid=0x00007fd8c00e8800 nid=0x1491b waiting on condition [0x00007000084ca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007990d02e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007fd8c009a000 nid=0x11813 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007fd8c0099800 nid=0x15707 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007fd8bf9fa000 nid=0x15507 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #403 daemon prio=5 os_prio=31 tid=0x00007fd8be875800 nid=0x15307 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #402 daemon prio=5 os_prio=31 tid=0x00007fd8bf862000 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #400 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1800 nid=0x1510f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796a4fd60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #399 daemon prio=5 os_prio=31 tid=0x00007fd8be83a000 nid=0x11213 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000796e4c318> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #398 daemon prio=5 os_prio=31 tid=0x00007fd8c00d2800 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000797054e48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #397 daemon prio=5 os_prio=31 tid=0x00007fd8ba320800 nid=0x991f in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000797032d30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #395 daemon prio=5 os_prio=31 tid=0x00007fd8ba294800 nid=0xfb0f in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e83690> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #394 daemon prio=5 os_prio=31 tid=0x00007fd8be813000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e57e00> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #393 daemon prio=5 os_prio=31 tid=0x00007fd8bea27000 nid=0x11d13 in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e55428> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #392 daemon prio=5 os_prio=31 tid=0x00007fd8bea1b000 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000796e52c70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #391 daemon prio=5 os_prio=31 tid=0x00007fd8bf947800 nid=0xfc1b runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079701a1e8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796e50e00> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x0000000796a53078> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #377 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1000 nid=0xee17 in Object.wait() [0x0000700008de5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.util.TimerThread.mainLoop(Timer.java:526) - locked <0x0000000795ac4a40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #374 daemon prio=5 os_prio=31 tid=0x00007fd8bf8a8800 nid=0xf20f in Object.wait() [0x0000700008bdf000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007958f3248> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #353 daemon prio=5 os_prio=31 tid=0x00007fd8c00e0000 nid=0x14107 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a098b480> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007fd8ba42d000 nid=0x13703 in Object.wait() [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a096b618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cf000 nid=0x11f0b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a09f81c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4da8a050" #306 daemon prio=5 os_prio=31 tid=0x00007fd8be812800 nid=0x10f07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007fd8ba436800 nid=0x10d07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007fd8ba436000 nid=0x10b07 waiting on condition [0x0000700007fbb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007fd8ba435000 nid=0xf70f waiting on condition [0x0000700007eb8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0aa6d50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007fd8bf962000 nid=0x1090f waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007fd8ba2ae000 nid=0x10707 waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007fd8c019d800 nid=0x10507 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007fd8c019d000 nid=0x10307 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007fd8c019c000 nid=0x10107 waiting on condition [0x00007000079a9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007fd8bf961800 nid=0xff07 waiting on condition [0x00007000078a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09f9110> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007fd8be811800 nid=0x3e0b in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aaa9d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007fd8c019a800 nid=0x6107 in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007fd8be811000 nid=0x5d07 in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acd940> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007fd8be810000 nid=0x7b17 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aab628> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007fd8be83b000 nid=0xec0b in Object.wait() [0x0000700000e3d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aabe58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007fd8bf861000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0abf2d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007fd8bf865800 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0aceba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f9000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0acf3a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1f800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0891c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4323fbfd" #221 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1e000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007fd8be80d800 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007fd8ba291000 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007fd8ba290800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0a902b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007fd8bc1b1800 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007fd8bea57800 nid=0xde03 waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1d000 nid=0x880f waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007fd8bea56800 nid=0x3c0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007fd8bc8c4000 nid=0x8a0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f3800 nid=0x8d0f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a0892de8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007fd8ba69f800 nid=0x840f in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0895c48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007fd8bbc71000 nid=0xa01b in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2dad8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007fd8bea59000 nid=0x7e0f in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2f278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a92cb0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007fd8ba5ed000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2e2a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007fd8be9f3800 nid=0xaa0b in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a0a2ead0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007fd8bbb75800 nid=0x760f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740ae72c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@5ab03154" #198 daemon prio=5 os_prio=31 tid=0x00007fd8ba2b0000 nid=0x690f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007fd8ba2af000 nid=0x670f waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007fd8bf98f000 nid=0x7217 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007fd8bf98e800 nid=0xc80f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a08d19d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007fd8ba279800 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007fd8bbc5a000 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007fd8ba279000 nid=0xd807 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007fd8c0055000 nid=0xc407 waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007fd8bf8bd000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007fd8ba351000 nid=0xd40f waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007a09056d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007fd8ba218800 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740a8fd70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007fd8ba218000 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007a08911a0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007fd8bbcae800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91130> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007fd8bbdb5000 nid=0xca07 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a917f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007fd8ba33c000 nid=0xc007 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007fd8ba25b000 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007fd8ba136800 nid=0xbc07 in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007fd8bea58000 nid=0xc203 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740accbf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@76053569" #151 daemon prio=5 os_prio=31 tid=0x00007fd8ba25a000 nid=0xba03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f3800 nid=0xb803 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007fd8bea56000 nid=0xb603 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007fd8bea54800 nid=0xb40b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acd3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007fd8bfa7f000 nid=0xb20b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740b2e648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007fd8bbbcb800 nid=0xb003 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007fd8be809800 nid=0x6217 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f2800 nid=0x9e0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007fd8bbde1800 nid=0x660b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007fd8bbb57000 nid=0x740b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007fd8bbb7b800 nid=0x960b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acf0a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c6800 nid=0xa813 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc3ed0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c5800 nid=0xa60b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be3f68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007fd8bea54000 nid=0xa40b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b800 nid=0x920b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b000 nid=0xa213 in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007fd8ba1b5800 nid=0x6e0b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b2dd50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cc800 nid=0x8213 in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be5548> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007fd8bf8c8800 nid=0x7807 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cb9400> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007fd8baaea000 nid=0x8b07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cc5dc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007fd8bac06800 nid=0x9003 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740aa5078> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007fd8ba361000 nid=0x8003 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740b088f0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007fd8baa09000 nid=0x6a03 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000074001bf40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #12 daemon prio=10 os_prio=31 tid=0x00007fd8ba9c9800 nid=0x5b03 waiting on condition [0x0000700001786000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c440> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007fd8b9904000 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007400375a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007fd8b98f8800 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007400317a0> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007fd8bb80d000 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007fd8ba028800 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007fd8ba822800 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007fd8bb800800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007fd8b985e800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007fd8b9840800 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007fd8b984c000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007fd8b980c800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007fd8bb802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x0000000796a531d0> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007fd8ba80a800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813000 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813800 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007fd8b9814000 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007fd8bb80a800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007fd8ba002000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003000 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003800 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007fd8ba004000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007fd8bb801000 nid=0x5503 waiting on condition JNI global references: 301 2019-02-18 13:17:53 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #413 daemon prio=9 os_prio=31 tid=0x00007fd8bc8d8000 nid=0x1450b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@359f04ca" #412 daemon prio=5 os_prio=31 tid=0x00007fd8ba4e7000 nid=0x1360b waiting on condition [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #411 daemon prio=5 os_prio=31 tid=0x00007fd8c00e9800 nid=0x13f0b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #410 daemon prio=5 os_prio=31 tid=0x00007fd8bf868800 nid=0x1340b waiting on condition [0x00007000085cd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #409 daemon prio=5 os_prio=31 tid=0x00007fd8c00e8800 nid=0x1491b waiting on condition [0x00007000084ca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079f96eaa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007fd8c009a000 nid=0x11813 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007fd8c0099800 nid=0x15707 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007fd8bf9fa000 nid=0x15507 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #403 daemon prio=5 os_prio=31 tid=0x00007fd8be875800 nid=0x15307 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #402 daemon prio=5 os_prio=31 tid=0x00007fd8bf862000 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #400 daemon prio=5 os_prio=31 tid=0x00007fd8c00d1800 nid=0x1510f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079f9889b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #399 daemon prio=5 os_prio=31 tid=0x00007fd8be83a000 nid=0x11213 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079f816b48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #397 daemon prio=5 os_prio=31 tid=0x00007fd8ba320800 nid=0x991f in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f920460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #395 daemon prio=5 os_prio=31 tid=0x00007fd8ba294800 nid=0xfb0f in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f921920> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #394 daemon prio=5 os_prio=31 tid=0x00007fd8be813000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f998ff0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #393 daemon prio=5 os_prio=31 tid=0x00007fd8bea27000 nid=0x11d13 in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f920b50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #392 daemon prio=5 os_prio=31 tid=0x00007fd8bea1b000 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f921270> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #391 daemon prio=5 os_prio=31 tid=0x00007fd8bf947800 nid=0xfc1b runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079f8176c0> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079f817510> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x000000079f8170a0> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079f817228> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-85" #374 daemon prio=5 os_prio=31 tid=0x00007fd8bf8a8800 nid=0xf20f in Object.wait() [0x0000700008bdf000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f99e608> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #353 daemon prio=5 os_prio=31 tid=0x00007fd8c00e0000 nid=0x14107 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f98aba0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007fd8ba42d000 nid=0x13703 in Object.wait() [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f98b120> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cf000 nid=0x11f0b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079fa58200> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4da8a050" #306 daemon prio=5 os_prio=31 tid=0x00007fd8be812800 nid=0x10f07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007fd8ba436800 nid=0x10d07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007fd8ba436000 nid=0x10b07 waiting on condition [0x0000700007fbb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007fd8ba435000 nid=0xf70f waiting on condition [0x0000700007eb8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079fa589c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007fd8bf962000 nid=0x1090f waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007fd8ba2ae000 nid=0x10707 waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007fd8c019d800 nid=0x10507 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007fd8c019d000 nid=0x10307 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007fd8c019c000 nid=0x10107 waiting on condition [0x00007000079a9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007fd8bf961800 nid=0xff07 waiting on condition [0x00007000078a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079fa58fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007fd8be811800 nid=0x3e0b in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079fabddf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007fd8c019a800 nid=0x6107 in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079fabe4a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007fd8be811000 nid=0x5d07 in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079fabea48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007fd8be810000 nid=0x7b17 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079fabf158> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007fd8be83b000 nid=0xec0b in Object.wait() [0x0000700000e3d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079fabf898> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007fd8bf861000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079faed768> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007fd8bf865800 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f9ac5d0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f9000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f9acb50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1f800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079f809c20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@4323fbfd" #221 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1e000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007fd8be80d800 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007fd8ba291000 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007fd8ba290800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079fa60578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007fd8bc1b1800 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007fd8bea57800 nid=0xde03 waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007fd8bbe1d000 nid=0x880f waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007fd8bea56800 nid=0x3c0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007fd8bc8c4000 nid=0x8a0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f3800 nid=0x8d0f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079f80add8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007fd8ba69f800 nid=0x840f in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000079f80ec40> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f80ec40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007fd8bbc71000 nid=0xa01b in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f9b9f40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007fd8bea59000 nid=0x7e0f in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f9bb510> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007fd8bf9f4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000079fa62f78> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079fa62f78> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007fd8ba5ed000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f9ba650> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007fd8be9f3800 nid=0xaa0b in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f9bad90> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007fd8bbb75800 nid=0x760f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740ae72c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@5ab03154" #198 daemon prio=5 os_prio=31 tid=0x00007fd8ba2b0000 nid=0x690f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007fd8ba2af000 nid=0x670f waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007fd8bf98f000 nid=0x7217 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007fd8bf98e800 nid=0xc80f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079f9044a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007fd8ba279800 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007fd8bbc5a000 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007fd8ba279000 nid=0xd807 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007fd8c0055000 nid=0xc407 waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007fd8bf8bd000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007fd8ba351000 nid=0xd40f waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079f908180> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007fd8ba218800 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740a8fd70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007fd8ba218000 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000079f809190> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079f809190> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007fd8bbcae800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91130> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007fd8bbdb5000 nid=0xca07 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a917f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007fd8ba33c000 nid=0xc007 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a91ca0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007fd8ba25b000 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a92148> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007fd8ba136800 nid=0xbc07 in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740a925f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007fd8bea58000 nid=0xc203 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740accbf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@76053569" #151 daemon prio=5 os_prio=31 tid=0x00007fd8ba25a000 nid=0xba03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f3800 nid=0xb803 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007fd8bea56000 nid=0xb603 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007fd8bea54800 nid=0xb40b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acd3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007fd8bfa7f000 nid=0xb20b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740b2e648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007fd8bbbcb800 nid=0xb003 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007fd8be809800 nid=0x6217 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007fd8ba4f2800 nid=0x9e0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007fd8bbde1800 nid=0x660b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007fd8bbb57000 nid=0x740b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007fd8bbb7b800 nid=0x960b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740acf0a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c6800 nid=0xa813 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc3ed0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007fd8bf9c5800 nid=0xa60b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be3f68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007fd8bea54000 nid=0xa40b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740bc4560> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b800 nid=0x920b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be44e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007fd8bea5b000 nid=0xa213 in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be4cf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007fd8ba1b5800 nid=0x6e0b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b2dd50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007fd8bf8cc800 nid=0x8213 in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740be5548> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007fd8bf8c8800 nid=0x7807 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cb9400> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007fd8baaea000 nid=0x8b07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cc5dc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007fd8bac06800 nid=0x9003 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740aa5078> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007fd8ba361000 nid=0x8003 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740b088f0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007fd8baa09000 nid=0x6a03 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000074001bf40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #12 daemon prio=10 os_prio=31 tid=0x00007fd8ba9c9800 nid=0x5b03 waiting on condition [0x0000700001786000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c440> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007fd8b9904000 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007400375a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007fd8b98f8800 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007400317a0> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007fd8bb80d000 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007fd8ba028800 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007fd8ba822800 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007fd8bb800800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007fd8b985e800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007fd8b9840800 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007fd8b984c000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740031978> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007fd8b980c800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x000000074003f930> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007fd8bb802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x000000079f817228> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007fd8ba80a800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813000 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007fd8b9813800 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007fd8b9814000 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007fd8bb80a800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007fd8ba002000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003000 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007fd8ba003800 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007fd8ba004000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007fd8bb801000 nid=0x5503 waiting on condition JNI global references: 307