2019-02-18 13:23:23 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #414 daemon prio=9 os_prio=31 tid=0x00007f8d2f80f000 nid=0x5b07 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@55ce3f49" #413 daemon prio=5 os_prio=31 tid=0x00007f8d2b9dd000 nid=0x1430b waiting on condition [0x00007000089d9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #412 daemon prio=5 os_prio=31 tid=0x00007f8d2dd0a800 nid=0x1380b waiting on condition [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #411 daemon prio=5 os_prio=31 tid=0x00007f8d2f03b000 nid=0x1420b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #410 daemon prio=5 os_prio=31 tid=0x00007f8d2f80c000 nid=0x14c1b waiting on condition [0x00007000085cd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079d6c3c28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #407 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a0000 nid=0x11813 waiting on condition [0x00007000084ca000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007f8d2d967800 nid=0x15c07 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007f8d2f063000 nid=0x15a07 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007f8d2f99f000 nid=0x15807 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #403 daemon prio=5 os_prio=31 tid=0x00007f8d2f99a800 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #401 daemon prio=5 os_prio=31 tid=0x00007f8d2d915800 nid=0x1560f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b302210> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #400 daemon prio=5 os_prio=31 tid=0x00007f8d2d841000 nid=0x11013 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b6fd7e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #399 daemon prio=5 os_prio=31 tid=0x00007f8d30099000 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b9060f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #398 daemon prio=5 os_prio=31 tid=0x00007f8d30174800 nid=0x1320b in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b8e3ff0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #396 daemon prio=5 os_prio=31 tid=0x00007f8d30164800 nid=0x1300b in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b734b40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #395 daemon prio=5 os_prio=31 tid=0x00007f8d3015c000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7092b0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #394 daemon prio=5 os_prio=31 tid=0x00007f8d2f961000 nid=0xfd1b in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7068d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #393 daemon prio=5 os_prio=31 tid=0x00007f8d2f8c6800 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b704138> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #392 daemon prio=5 os_prio=31 tid=0x00007f8d2f041800 nid=0x11c13 runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b8cb4b8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b7022c8> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x000000079b305528> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #378 daemon prio=5 os_prio=31 tid=0x00007f8d2d978800 nid=0xef17 in Object.wait() [0x0000700008ee8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079a10e278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #375 daemon prio=5 os_prio=31 tid=0x00007f8d3015b000 nid=0xf20f in Object.wait() [0x0000700008ce2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000799f3ccf0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-81" #357 daemon prio=5 os_prio=31 tid=0x00007f8d2f99e800 nid=0x14807 in Object.wait() [0x0000700006b7f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795da1a08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #354 daemon prio=5 os_prio=31 tid=0x00007f8d2f99c000 nid=0x14407 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795b174a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #344 daemon prio=10 os_prio=31 tid=0x00007f8d2b863000 nid=0x14e03 waiting on condition [0x0000700000de8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c7e0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-76" #335 daemon prio=5 os_prio=31 tid=0x00007f8d3017c000 nid=0x13e03 in Object.wait() [0x0000700008adc000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2401a50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007f8d2f177000 nid=0x13a03 in Object.wait() [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2430180> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-71" #314 daemon prio=5 os_prio=31 tid=0x00007f8d30164000 nid=0x12a0b in Object.wait() [0x0000700006979000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b24284c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a5000 nid=0x1150b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2432f70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@ba952cf" #306 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1800 nid=0x9717 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbb000 nid=0xfa07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007f8d2ddba000 nid=0x10e07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007f8d2ddb9800 nid=0x10c0f waiting on condition [0x0000700007fbb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b24f65f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007f8d2f06b800 nid=0x10a07 waiting on condition [0x0000700007eb8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007f8d2da22000 nid=0xf607 waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007f8d2d129000 nid=0x1080f waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007f8d2d8d4000 nid=0x10607 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007f8d3017b000 nid=0x10407 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007f8d2f196800 nid=0x10207 waiting on condition [0x00007000079a9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b247b708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-66" #293 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1000 nid=0x10007 in Object.wait() [0x00007000078a6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2553d80> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b0000 nid=0xfe07 in Object.wait() [0x00007000077a3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25275d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007f8d2f9af800 nid=0x7813 in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2527c88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ae800 nid=0xec0b in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2578240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007f8d30186000 nid=0x3c0b in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570378> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007f8d2f97e800 nid=0x5f07 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570ba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-61" #272 daemon prio=5 os_prio=31 tid=0x00007f8d2b861800 nid=0x12407 in Object.wait() [0x0000700006773000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25713f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007f8d2f078000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25789c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-56" #251 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbc000 nid=0x11a03 in Object.wait() [0x00007000076a0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2564d70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007f8d2d915000 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25653e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-51" #230 daemon prio=5 os_prio=31 tid=0x00007f8d2f057000 nid=0xf803 in Object.wait() [0x000070000656d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2565968> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007f8d2f13a000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25b2fc8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b256bd38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@38fa1414" #221 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007f8d2f97c000 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007f8d2b8b2800 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007f8d2f97b800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b25ddc88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007f8d2b824000 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007f8d2d291800 nid=0x720f waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007f8d2f139000 nid=0xa217 waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007f8d2f813800 nid=0x8d0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007f8d2f812800 nid=0x6c0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ba000 nid=0x860f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b2423c60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b9800 nid=0x8a17 in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-45" #207 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8800 nid=0xa01b in Object.wait() [0x0000700005234000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25de4c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007f8d30160000 nid=0x8217 in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256e460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007f8d3015f800 nid=0xb00b in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256f060> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007f8d300a4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b7000 nid=0x7c0f in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007f8d300ae800 nid=0x680f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d3d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@41f61dd3" #198 daemon prio=5 os_prio=31 tid=0x00007f8d2f800000 nid=0x760f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007f8d2f044800 nid=0xaa13 waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007f8d2f044000 nid=0xca07 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007f8d2f043000 nid=0xde0f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c29380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007f8d2f134000 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007f8d2f133800 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007f8d2f96a800 nid=0xc607 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007f8d3009c000 nid=0xd80f waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007f8d3009b000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007f8d2e9d5000 nid=0xd407 waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c67758> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007f8d2b39d000 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d4eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007f8d2b227800 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-39" #183 daemon prio=5 os_prio=31 tid=0x00007f8d2c7f4800 nid=0xce07 in Object.wait() [0x00007000025b0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007f8d2b28e800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba5f88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007f8d2c338800 nid=0xc007 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba6618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007f8d2c3c1800 nid=0xc207 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007f8d2c5bd800 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007f8d2c426800 nid=0x3e0b in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-34" #158 daemon prio=5 os_prio=31 tid=0x00007f8d2b227000 nid=0xc703 in Object.wait() [0x0000700004c22000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007f8d2b0f7800 nid=0xc303 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740caabc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@1a795765" #151 daemon prio=5 os_prio=31 tid=0x00007f8d2c0a6000 nid=0xbb03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007f8d2ea32800 nid=0xb903 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007f8d2c2de800 nid=0xb703 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007f8d2c3ca800 nid=0xb50b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d043f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007f8d2e996000 nid=0xb30b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d061e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007f8d2e995800 nid=0xb103 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e7000 nid=0x6117 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007f8d2ebf6800 nid=0x9d0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e6000 nid=0x650b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007f8d2ea31000 nid=0x630b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007f8d3015e800 nid=0x730b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740cab540> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-29" #135 daemon prio=5 os_prio=31 tid=0x00007f8d2b1eb000 nid=0x990b in Object.wait() [0x0000700003aef000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007f8d2db8f800 nid=0x9513 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91488> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007f8d2d90f800 nid=0xa70b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d99c60> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007f8d2d85b000 nid=0xa50b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007f8d2d91a800 nid=0xa30b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007f8d2d894800 nid=0x910b in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-24" #110 daemon prio=5 os_prio=31 tid=0x00007f8d2d879000 nid=0x870b in Object.wait() [0x00007000023aa000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007f8d2e99a000 nid=0x830b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9aa30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007f8d2b38f800 nid=0x6d0b in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e898> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-19" #88 daemon prio=5 os_prio=31 tid=0x00007f8d2ea02800 nid=0x7d07 in Object.wait() [0x00007000035e0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007f8d2b28b000 nid=0x7907 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da5900> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-14" #67 daemon prio=5 os_prio=31 tid=0x00007f8d2ec87800 nid=0x9b07 in Object.wait() [0x00007000021a4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007f8d2c43a800 nid=0x8c07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e2d760> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-9" #45 daemon prio=5 os_prio=31 tid=0x00007f8d2c487800 nid=0x9303 in Object.wait() [0x00007000031d4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007f8d2d96d000 nid=0x8f03 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b41e10> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007f8d2b243800 nid=0x7f03 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-4" #22 daemon prio=5 os_prio=31 tid=0x00007f8d2b28d800 nid=0x6f03 in Object.wait() [0x0000700001f9e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007f8d2ea86000 nid=0x6903 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740014bc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007f8d2d842800 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740014f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007f8d2d025000 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x0000000740022400> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f8d2c009800 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f8d2c014000 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f8d2c013000 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f8d2c012800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f8d2c010800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f8d2d848000 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f8d2e800000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740022750> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f8d2c00f800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x0000000740022920> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007f8d2d802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007f8d2d811800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f8d2e004800 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002000 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002800 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f8d2c006800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f8d2c00d000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f8d2b801800 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f8d2b802000 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f8d2d003000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007f8d2c037000 nid=0x5503 waiting on condition JNI global references: 312 2019-02-18 13:23:25 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #414 daemon prio=9 os_prio=31 tid=0x00007f8d2f80f000 nid=0x5b07 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@55ce3f49" #413 daemon prio=5 os_prio=31 tid=0x00007f8d2b9dd000 nid=0x1430b sleeping[0x00007000089d9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #412 daemon prio=5 os_prio=31 tid=0x00007f8d2dd0a800 nid=0x1380b waiting on condition [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #411 daemon prio=5 os_prio=31 tid=0x00007f8d2f03b000 nid=0x1420b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #410 daemon prio=5 os_prio=31 tid=0x00007f8d2f80c000 nid=0x14c1b waiting on condition [0x00007000085cd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079d6c3c28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #407 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a0000 nid=0x11813 waiting on condition [0x00007000084ca000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007f8d2d967800 nid=0x15c07 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007f8d2f063000 nid=0x15a07 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007f8d2f99f000 nid=0x15807 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #403 daemon prio=5 os_prio=31 tid=0x00007f8d2f99a800 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #401 daemon prio=5 os_prio=31 tid=0x00007f8d2d915800 nid=0x1560f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b302210> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #400 daemon prio=5 os_prio=31 tid=0x00007f8d2d841000 nid=0x11013 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b6fd7e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #399 daemon prio=5 os_prio=31 tid=0x00007f8d30099000 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b9060f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #398 daemon prio=5 os_prio=31 tid=0x00007f8d30174800 nid=0x1320b in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b8e3ff0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #396 daemon prio=5 os_prio=31 tid=0x00007f8d30164800 nid=0x1300b in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b734b40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #395 daemon prio=5 os_prio=31 tid=0x00007f8d3015c000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7092b0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #394 daemon prio=5 os_prio=31 tid=0x00007f8d2f961000 nid=0xfd1b in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7068d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #393 daemon prio=5 os_prio=31 tid=0x00007f8d2f8c6800 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b704138> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #392 daemon prio=5 os_prio=31 tid=0x00007f8d2f041800 nid=0x11c13 runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b8cb4b8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b7022c8> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x000000079b305528> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #378 daemon prio=5 os_prio=31 tid=0x00007f8d2d978800 nid=0xef17 in Object.wait() [0x0000700008ee8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079a10e278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #375 daemon prio=5 os_prio=31 tid=0x00007f8d3015b000 nid=0xf20f in Object.wait() [0x0000700008ce2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000799f3ccf0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-81" #357 daemon prio=5 os_prio=31 tid=0x00007f8d2f99e800 nid=0x14807 in Object.wait() [0x0000700006b7f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795da1a08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #354 daemon prio=5 os_prio=31 tid=0x00007f8d2f99c000 nid=0x14407 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795b174a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #344 daemon prio=10 os_prio=31 tid=0x00007f8d2b863000 nid=0x14e03 waiting on condition [0x0000700000de8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c7e0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-76" #335 daemon prio=5 os_prio=31 tid=0x00007f8d3017c000 nid=0x13e03 in Object.wait() [0x0000700008adc000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2401a50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007f8d2f177000 nid=0x13a03 in Object.wait() [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2430180> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-71" #314 daemon prio=5 os_prio=31 tid=0x00007f8d30164000 nid=0x12a0b in Object.wait() [0x0000700006979000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b24284c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a5000 nid=0x1150b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2432f70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@ba952cf" #306 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1800 nid=0x9717 sleeping[0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbb000 nid=0xfa07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007f8d2ddba000 nid=0x10e07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007f8d2ddb9800 nid=0x10c0f waiting on condition [0x0000700007fbb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b24f65f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007f8d2f06b800 nid=0x10a07 waiting on condition [0x0000700007eb8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007f8d2da22000 nid=0xf607 waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007f8d2d129000 nid=0x1080f waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007f8d2d8d4000 nid=0x10607 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007f8d3017b000 nid=0x10407 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007f8d2f196800 nid=0x10207 waiting on condition [0x00007000079a9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b247b708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-66" #293 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1000 nid=0x10007 in Object.wait() [0x00007000078a6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2553d80> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b0000 nid=0xfe07 in Object.wait() [0x00007000077a3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25275d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007f8d2f9af800 nid=0x7813 in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2527c88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ae800 nid=0xec0b in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2578240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007f8d30186000 nid=0x3c0b in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570378> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007f8d2f97e800 nid=0x5f07 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570ba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-61" #272 daemon prio=5 os_prio=31 tid=0x00007f8d2b861800 nid=0x12407 in Object.wait() [0x0000700006773000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25713f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007f8d2f078000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25789c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-56" #251 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbc000 nid=0x11a03 in Object.wait() [0x00007000076a0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2564d70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007f8d2d915000 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25653e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-51" #230 daemon prio=5 os_prio=31 tid=0x00007f8d2f057000 nid=0xf803 in Object.wait() [0x000070000656d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2565968> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007f8d2f13a000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25b2fc8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b256bd38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@38fa1414" #221 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007f8d2f97c000 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007f8d2b8b2800 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007f8d2f97b800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b25ddc88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007f8d2b824000 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007f8d2d291800 nid=0x720f waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007f8d2f139000 nid=0xa217 waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007f8d2f813800 nid=0x8d0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007f8d2f812800 nid=0x6c0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ba000 nid=0x860f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b2423c60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b9800 nid=0x8a17 in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-45" #207 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8800 nid=0xa01b in Object.wait() [0x0000700005234000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25de4c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007f8d30160000 nid=0x8217 in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256e460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007f8d3015f800 nid=0xb00b in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256f060> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007f8d300a4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b7000 nid=0x7c0f in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007f8d300ae800 nid=0x680f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d3d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@41f61dd3" #198 daemon prio=5 os_prio=31 tid=0x00007f8d2f800000 nid=0x760f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007f8d2f044800 nid=0xaa13 waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007f8d2f044000 nid=0xca07 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007f8d2f043000 nid=0xde0f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c29380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007f8d2f134000 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007f8d2f133800 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007f8d2f96a800 nid=0xc607 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007f8d3009c000 nid=0xd80f waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007f8d3009b000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007f8d2e9d5000 nid=0xd407 waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c67758> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007f8d2b39d000 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d4eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007f8d2b227800 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-39" #183 daemon prio=5 os_prio=31 tid=0x00007f8d2c7f4800 nid=0xce07 in Object.wait() [0x00007000025b0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007f8d2b28e800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba5f88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007f8d2c338800 nid=0xc007 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba6618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007f8d2c3c1800 nid=0xc207 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007f8d2c5bd800 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007f8d2c426800 nid=0x3e0b in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-34" #158 daemon prio=5 os_prio=31 tid=0x00007f8d2b227000 nid=0xc703 in Object.wait() [0x0000700004c22000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007f8d2b0f7800 nid=0xc303 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740caabc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@1a795765" #151 daemon prio=5 os_prio=31 tid=0x00007f8d2c0a6000 nid=0xbb03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007f8d2ea32800 nid=0xb903 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007f8d2c2de800 nid=0xb703 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007f8d2c3ca800 nid=0xb50b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d043f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007f8d2e996000 nid=0xb30b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d061e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007f8d2e995800 nid=0xb103 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e7000 nid=0x6117 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007f8d2ebf6800 nid=0x9d0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e6000 nid=0x650b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007f8d2ea31000 nid=0x630b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007f8d3015e800 nid=0x730b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740cab540> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-29" #135 daemon prio=5 os_prio=31 tid=0x00007f8d2b1eb000 nid=0x990b in Object.wait() [0x0000700003aef000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007f8d2db8f800 nid=0x9513 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91488> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007f8d2d90f800 nid=0xa70b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d99c60> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007f8d2d85b000 nid=0xa50b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007f8d2d91a800 nid=0xa30b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007f8d2d894800 nid=0x910b in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-24" #110 daemon prio=5 os_prio=31 tid=0x00007f8d2d879000 nid=0x870b in Object.wait() [0x00007000023aa000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007f8d2e99a000 nid=0x830b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9aa30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007f8d2b38f800 nid=0x6d0b in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e898> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-19" #88 daemon prio=5 os_prio=31 tid=0x00007f8d2ea02800 nid=0x7d07 in Object.wait() [0x00007000035e0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007f8d2b28b000 nid=0x7907 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da5900> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-14" #67 daemon prio=5 os_prio=31 tid=0x00007f8d2ec87800 nid=0x9b07 in Object.wait() [0x00007000021a4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007f8d2c43a800 nid=0x8c07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e2d760> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-9" #45 daemon prio=5 os_prio=31 tid=0x00007f8d2c487800 nid=0x9303 in Object.wait() [0x00007000031d4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007f8d2d96d000 nid=0x8f03 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b41e10> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007f8d2b243800 nid=0x7f03 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-4" #22 daemon prio=5 os_prio=31 tid=0x00007f8d2b28d800 nid=0x6f03 in Object.wait() [0x0000700001f9e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007f8d2ea86000 nid=0x6903 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740014bc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007f8d2d842800 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740014f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007f8d2d025000 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x0000000740022400> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f8d2c009800 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f8d2c014000 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f8d2c013000 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f8d2c012800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f8d2c010800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f8d2d848000 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f8d2e800000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740022750> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f8d2c00f800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x0000000740022920> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007f8d2d802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007f8d2d811800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f8d2e004800 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002000 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002800 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f8d2c006800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f8d2c00d000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f8d2b801800 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f8d2b802000 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f8d2d003000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007f8d2c037000 nid=0x5503 waiting on condition JNI global references: 312 2019-02-18 13:23:26 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #414 daemon prio=9 os_prio=31 tid=0x00007f8d2f80f000 nid=0x5b07 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@55ce3f49" #413 daemon prio=5 os_prio=31 tid=0x00007f8d2b9dd000 nid=0x1430b waiting on condition [0x00007000089d9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #412 daemon prio=5 os_prio=31 tid=0x00007f8d2dd0a800 nid=0x1380b waiting on condition [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #411 daemon prio=5 os_prio=31 tid=0x00007f8d2f03b000 nid=0x1420b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #410 daemon prio=5 os_prio=31 tid=0x00007f8d2f80c000 nid=0x14c1b waiting on condition [0x00007000085cd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079d6c3c28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #407 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a0000 nid=0x11813 waiting on condition [0x00007000084ca000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007f8d2d967800 nid=0x15c07 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007f8d2f063000 nid=0x15a07 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007f8d2f99f000 nid=0x15807 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #403 daemon prio=5 os_prio=31 tid=0x00007f8d2f99a800 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #401 daemon prio=5 os_prio=31 tid=0x00007f8d2d915800 nid=0x1560f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b302210> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #400 daemon prio=5 os_prio=31 tid=0x00007f8d2d841000 nid=0x11013 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b6fd7e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #399 daemon prio=5 os_prio=31 tid=0x00007f8d30099000 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b9060f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #398 daemon prio=5 os_prio=31 tid=0x00007f8d30174800 nid=0x1320b in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b8e3ff0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #396 daemon prio=5 os_prio=31 tid=0x00007f8d30164800 nid=0x1300b in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b734b40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #395 daemon prio=5 os_prio=31 tid=0x00007f8d3015c000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7092b0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #394 daemon prio=5 os_prio=31 tid=0x00007f8d2f961000 nid=0xfd1b in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7068d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #393 daemon prio=5 os_prio=31 tid=0x00007f8d2f8c6800 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b704138> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #392 daemon prio=5 os_prio=31 tid=0x00007f8d2f041800 nid=0x11c13 runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b8cb4b8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b7022c8> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x000000079b305528> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #378 daemon prio=5 os_prio=31 tid=0x00007f8d2d978800 nid=0xef17 in Object.wait() [0x0000700008ee8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079a10e278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #375 daemon prio=5 os_prio=31 tid=0x00007f8d3015b000 nid=0xf20f in Object.wait() [0x0000700008ce2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000799f3ccf0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-81" #357 daemon prio=5 os_prio=31 tid=0x00007f8d2f99e800 nid=0x14807 in Object.wait() [0x0000700006b7f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795da1a08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #354 daemon prio=5 os_prio=31 tid=0x00007f8d2f99c000 nid=0x14407 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795b174a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #344 daemon prio=10 os_prio=31 tid=0x00007f8d2b863000 nid=0x14e03 waiting on condition [0x0000700000de8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c7e0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-76" #335 daemon prio=5 os_prio=31 tid=0x00007f8d3017c000 nid=0x13e03 in Object.wait() [0x0000700008adc000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2401a50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007f8d2f177000 nid=0x13a03 in Object.wait() [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2430180> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-71" #314 daemon prio=5 os_prio=31 tid=0x00007f8d30164000 nid=0x12a0b in Object.wait() [0x0000700006979000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b24284c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a5000 nid=0x1150b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2432f70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@ba952cf" #306 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1800 nid=0x9717 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbb000 nid=0xfa07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007f8d2ddba000 nid=0x10e07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007f8d2ddb9800 nid=0x10c0f waiting on condition [0x0000700007fbb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b24f65f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007f8d2f06b800 nid=0x10a07 waiting on condition [0x0000700007eb8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007f8d2da22000 nid=0xf607 waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007f8d2d129000 nid=0x1080f waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007f8d2d8d4000 nid=0x10607 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007f8d3017b000 nid=0x10407 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007f8d2f196800 nid=0x10207 waiting on condition [0x00007000079a9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b247b708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-66" #293 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1000 nid=0x10007 in Object.wait() [0x00007000078a6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2553d80> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b0000 nid=0xfe07 in Object.wait() [0x00007000077a3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25275d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007f8d2f9af800 nid=0x7813 in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2527c88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ae800 nid=0xec0b in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2578240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007f8d30186000 nid=0x3c0b in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570378> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007f8d2f97e800 nid=0x5f07 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570ba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-61" #272 daemon prio=5 os_prio=31 tid=0x00007f8d2b861800 nid=0x12407 in Object.wait() [0x0000700006773000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25713f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007f8d2f078000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25789c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-56" #251 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbc000 nid=0x11a03 in Object.wait() [0x00007000076a0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2564d70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007f8d2d915000 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25653e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-51" #230 daemon prio=5 os_prio=31 tid=0x00007f8d2f057000 nid=0xf803 in Object.wait() [0x000070000656d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2565968> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007f8d2f13a000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25b2fc8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b256bd38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@38fa1414" #221 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007f8d2f97c000 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007f8d2b8b2800 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007f8d2f97b800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b25ddc88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007f8d2b824000 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007f8d2d291800 nid=0x720f waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007f8d2f139000 nid=0xa217 waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007f8d2f813800 nid=0x8d0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007f8d2f812800 nid=0x6c0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ba000 nid=0x860f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b2423c60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b9800 nid=0x8a17 in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-45" #207 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8800 nid=0xa01b in Object.wait() [0x0000700005234000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25de4c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007f8d30160000 nid=0x8217 in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256e460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007f8d3015f800 nid=0xb00b in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256f060> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007f8d300a4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b7000 nid=0x7c0f in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007f8d300ae800 nid=0x680f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d3d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@41f61dd3" #198 daemon prio=5 os_prio=31 tid=0x00007f8d2f800000 nid=0x760f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007f8d2f044800 nid=0xaa13 waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007f8d2f044000 nid=0xca07 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007f8d2f043000 nid=0xde0f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c29380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007f8d2f134000 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007f8d2f133800 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007f8d2f96a800 nid=0xc607 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007f8d3009c000 nid=0xd80f waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007f8d3009b000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007f8d2e9d5000 nid=0xd407 waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c67758> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007f8d2b39d000 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d4eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007f8d2b227800 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-39" #183 daemon prio=5 os_prio=31 tid=0x00007f8d2c7f4800 nid=0xce07 in Object.wait() [0x00007000025b0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007f8d2b28e800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba5f88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007f8d2c338800 nid=0xc007 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba6618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007f8d2c3c1800 nid=0xc207 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007f8d2c5bd800 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007f8d2c426800 nid=0x3e0b in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-34" #158 daemon prio=5 os_prio=31 tid=0x00007f8d2b227000 nid=0xc703 in Object.wait() [0x0000700004c22000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007f8d2b0f7800 nid=0xc303 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740caabc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@1a795765" #151 daemon prio=5 os_prio=31 tid=0x00007f8d2c0a6000 nid=0xbb03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007f8d2ea32800 nid=0xb903 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007f8d2c2de800 nid=0xb703 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007f8d2c3ca800 nid=0xb50b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d043f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007f8d2e996000 nid=0xb30b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d061e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007f8d2e995800 nid=0xb103 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e7000 nid=0x6117 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007f8d2ebf6800 nid=0x9d0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e6000 nid=0x650b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007f8d2ea31000 nid=0x630b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007f8d3015e800 nid=0x730b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740cab540> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-29" #135 daemon prio=5 os_prio=31 tid=0x00007f8d2b1eb000 nid=0x990b in Object.wait() [0x0000700003aef000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007f8d2db8f800 nid=0x9513 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91488> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007f8d2d90f800 nid=0xa70b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d99c60> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007f8d2d85b000 nid=0xa50b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007f8d2d91a800 nid=0xa30b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007f8d2d894800 nid=0x910b in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-24" #110 daemon prio=5 os_prio=31 tid=0x00007f8d2d879000 nid=0x870b in Object.wait() [0x00007000023aa000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007f8d2e99a000 nid=0x830b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9aa30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007f8d2b38f800 nid=0x6d0b in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e898> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-19" #88 daemon prio=5 os_prio=31 tid=0x00007f8d2ea02800 nid=0x7d07 in Object.wait() [0x00007000035e0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007f8d2b28b000 nid=0x7907 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da5900> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-14" #67 daemon prio=5 os_prio=31 tid=0x00007f8d2ec87800 nid=0x9b07 in Object.wait() [0x00007000021a4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007f8d2c43a800 nid=0x8c07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e2d760> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-9" #45 daemon prio=5 os_prio=31 tid=0x00007f8d2c487800 nid=0x9303 in Object.wait() [0x00007000031d4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007f8d2d96d000 nid=0x8f03 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b41e10> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007f8d2b243800 nid=0x7f03 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-4" #22 daemon prio=5 os_prio=31 tid=0x00007f8d2b28d800 nid=0x6f03 in Object.wait() [0x0000700001f9e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007f8d2ea86000 nid=0x6903 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740014bc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007f8d2d842800 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740014f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007f8d2d025000 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x0000000740022400> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f8d2c009800 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f8d2c014000 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f8d2c013000 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f8d2c012800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f8d2c010800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f8d2d848000 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f8d2e800000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740022750> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f8d2c00f800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x0000000740022920> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007f8d2d802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007f8d2d811800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f8d2e004800 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002000 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002800 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f8d2c006800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f8d2c00d000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f8d2b801800 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f8d2b802000 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f8d2d003000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007f8d2c037000 nid=0x5503 waiting on condition JNI global references: 312 2019-02-18 13:23:28 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): "Attach Listener" #414 daemon prio=9 os_prio=31 tid=0x00007f8d2f80f000 nid=0x5b07 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@55ce3f49" #413 daemon prio=5 os_prio=31 tid=0x00007f8d2b9dd000 nid=0x1430b waiting on condition [0x00007000089d9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #412 daemon prio=5 os_prio=31 tid=0x00007f8d2dd0a800 nid=0x1380b waiting on condition [0x00007000087d3000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #411 daemon prio=5 os_prio=31 tid=0x00007f8d2f03b000 nid=0x1420b waiting on condition [0x00007000086d0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #410 daemon prio=5 os_prio=31 tid=0x00007f8d2f80c000 nid=0x14c1b waiting on condition [0x00007000085cd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079d6c3c28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #407 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a0000 nid=0x11813 waiting on condition [0x00007000084ca000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #406 daemon prio=5 os_prio=31 tid=0x00007f8d2d967800 nid=0x15c07 waiting on condition [0x00007000083c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #405 daemon prio=5 os_prio=31 tid=0x00007f8d2f063000 nid=0x15a07 waiting on condition [0x0000700007294000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #404 daemon prio=5 os_prio=31 tid=0x00007f8d2f99f000 nid=0x15807 waiting on condition [0x0000700007191000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-357,5,FailOnTimeoutGroup]" #403 daemon prio=5 os_prio=31 tid=0x00007f8d2f99a800 nid=0xf313 waiting on condition [0x000070000708e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #401 daemon prio=5 os_prio=31 tid=0x00007f8d2d915800 nid=0x1560f waiting on condition [0x0000700006f8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b302210> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #400 daemon prio=5 os_prio=31 tid=0x00007f8d2d841000 nid=0x11013 waiting on condition [0x0000700006e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079b6fd7e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-91" #399 daemon prio=5 os_prio=31 tid=0x00007f8d30099000 nid=0x12c13 in Object.wait() [0x0000700006d85000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b9060f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #398 daemon prio=5 os_prio=31 tid=0x00007f8d30174800 nid=0x1320b in Object.wait() [0x0000700006c82000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b8e3ff0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-90" #396 daemon prio=5 os_prio=31 tid=0x00007f8d30164800 nid=0x1300b in Object.wait() [0x0000700006a7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b734b40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-89" #395 daemon prio=5 os_prio=31 tid=0x00007f8d3015c000 nid=0x12613 in Object.wait() [0x0000700006264000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7092b0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-88" #394 daemon prio=5 os_prio=31 tid=0x00007f8d2f961000 nid=0xfd1b in Object.wait() [0x0000700006161000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b7068d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-87" #393 daemon prio=5 os_prio=31 tid=0x00007f8d2f8c6800 nid=0x12213 in Object.wait() [0x000070000605e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079b704138> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Time-limited test" #392 daemon prio=5 os_prio=31 tid=0x00007f8d2f041800 nid=0x11c13 runnable [0x0000700001887000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart$6.handleStoreEvent(TestRMRestart.java:1650) at org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.storeProxyCACert(RMStateStore.java:1363) at org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager.serviceStart(ProxyCAManager.java:63) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b8cb4b8> (a java.lang.Object) at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:121) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$RMActiveServices.serviceStart(ResourceManager.java:914) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b7022c8> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.startActiveServices(ResourceManager.java:1280) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1321) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$1.run(ResourceManager.java:1317) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1876) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.transitionToActive(ResourceManager.java:1317) - locked <0x000000079b305528> (a org.apache.hadoop.yarn.server.resourcemanager.MockRM) at org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.serviceStart(ResourceManager.java:1368) at org.apache.hadoop.service.AbstractService.start(AbstractService.java:194) - locked <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.testRMStateStoreDispatcherDrainedOnRMStop(TestRMRestart.java:1660) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-86" #378 daemon prio=5 os_prio=31 tid=0x00007f8d2d978800 nid=0xef17 in Object.wait() [0x0000700008ee8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000079a10e278> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-85" #375 daemon prio=5 os_prio=31 tid=0x00007f8d3015b000 nid=0xf20f in Object.wait() [0x0000700008ce2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000799f3ccf0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-81" #357 daemon prio=5 os_prio=31 tid=0x00007f8d2f99e800 nid=0x14807 in Object.wait() [0x0000700006b7f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795da1a08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-80" #354 daemon prio=5 os_prio=31 tid=0x00007f8d2f99c000 nid=0x14407 in Object.wait() [0x0000700006876000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000795b174a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "process reaper" #344 daemon prio=10 os_prio=31 tid=0x00007f8d2b863000 nid=0x14e03 waiting on condition [0x0000700000de8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000074001c7e0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-76" #335 daemon prio=5 os_prio=31 tid=0x00007f8d3017c000 nid=0x13e03 in Object.wait() [0x0000700008adc000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2401a50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-75" #332 daemon prio=5 os_prio=31 tid=0x00007f8d2f177000 nid=0x13a03 in Object.wait() [0x00007000088d6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2430180> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-71" #314 daemon prio=5 os_prio=31 tid=0x00007f8d30164000 nid=0x12a0b in Object.wait() [0x0000700006979000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b24284c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-70" #311 daemon prio=5 os_prio=31 tid=0x00007f8d2f9a5000 nid=0x1150b in Object.wait() [0x0000700006670000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2432f70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@ba952cf" #306 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1800 nid=0x9717 waiting on condition [0x00007000082c4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #305 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbb000 nid=0xfa07 waiting on condition [0x00007000081c1000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #304 daemon prio=5 os_prio=31 tid=0x00007f8d2ddba000 nid=0x10e07 waiting on condition [0x00007000080be000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #303 daemon prio=5 os_prio=31 tid=0x00007f8d2ddb9800 nid=0x10c0f waiting on condition [0x0000700007fbb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b24f65f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #300 daemon prio=5 os_prio=31 tid=0x00007f8d2f06b800 nid=0x10a07 waiting on condition [0x0000700007eb8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #299 daemon prio=5 os_prio=31 tid=0x00007f8d2da22000 nid=0xf607 waiting on condition [0x0000700007db5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #298 daemon prio=5 os_prio=31 tid=0x00007f8d2d129000 nid=0x1080f waiting on condition [0x0000700007cb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #297 daemon prio=5 os_prio=31 tid=0x00007f8d2d8d4000 nid=0x10607 waiting on condition [0x0000700007baf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-259,5,FailOnTimeoutGroup]" #296 daemon prio=5 os_prio=31 tid=0x00007f8d3017b000 nid=0x10407 waiting on condition [0x0000700007aac000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #294 daemon prio=5 os_prio=31 tid=0x00007f8d2f196800 nid=0x10207 waiting on condition [0x00007000079a9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b247b708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-66" #293 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b1000 nid=0x10007 in Object.wait() [0x00007000078a6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2553d80> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #292 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b0000 nid=0xfe07 in Object.wait() [0x00007000077a3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25275d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-65" #290 daemon prio=5 os_prio=31 tid=0x00007f8d2f9af800 nid=0x7813 in Object.wait() [0x000070000759d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2527c88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-64" #289 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ae800 nid=0xec0b in Object.wait() [0x0000700007397000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2578240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-63" #288 daemon prio=5 os_prio=31 tid=0x00007f8d30186000 nid=0x3c0b in Object.wait() [0x0000700005f5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570378> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-62" #287 daemon prio=5 os_prio=31 tid=0x00007f8d2f97e800 nid=0x5f07 in Object.wait() [0x0000700004b1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2570ba8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-61" #272 daemon prio=5 os_prio=31 tid=0x00007f8d2b861800 nid=0x12407 in Object.wait() [0x0000700006773000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25713f8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-60" #269 daemon prio=5 os_prio=31 tid=0x00007f8d2f078000 nid=0x12007 in Object.wait() [0x000070000646a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25789c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-56" #251 daemon prio=5 os_prio=31 tid=0x00007f8d2ddbc000 nid=0x11a03 in Object.wait() [0x00007000076a0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2564d70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-55" #248 daemon prio=5 os_prio=31 tid=0x00007f8d2d915000 nid=0x11603 in Object.wait() [0x000070000749a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25653e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-51" #230 daemon prio=5 os_prio=31 tid=0x00007f8d2f057000 nid=0xf803 in Object.wait() [0x000070000656d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b2565968> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-50" #227 daemon prio=5 os_prio=31 tid=0x00007f8d2f13a000 nid=0xf403 in Object.wait() [0x0000700006367000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25b2fc8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #203 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d800 nid=0xea03 waiting on condition [0x0000700005e58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b256bd38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@38fa1414" #221 daemon prio=5 os_prio=31 tid=0x00007f8d2f97d000 nid=0xe803 waiting on condition [0x0000700005d55000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #220 daemon prio=5 os_prio=31 tid=0x00007f8d2f97c000 nid=0xe603 waiting on condition [0x0000700005c52000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #219 daemon prio=5 os_prio=31 tid=0x00007f8d2b8b2800 nid=0xe403 waiting on condition [0x0000700005b4f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #218 daemon prio=5 os_prio=31 tid=0x00007f8d2f97b800 nid=0xe20b waiting on condition [0x0000700005a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b25ddc88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #215 daemon prio=5 os_prio=31 tid=0x00007f8d2b824000 nid=0xe003 waiting on condition [0x0000700005949000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #214 daemon prio=5 os_prio=31 tid=0x00007f8d2d291800 nid=0x720f waiting on condition [0x0000700005846000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #213 daemon prio=5 os_prio=31 tid=0x00007f8d2f139000 nid=0xa217 waiting on condition [0x0000700005743000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #212 daemon prio=5 os_prio=31 tid=0x00007f8d2f813800 nid=0x8d0f waiting on condition [0x0000700005640000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-181,5,FailOnTimeoutGroup]" #211 daemon prio=5 os_prio=31 tid=0x00007f8d2f812800 nid=0x6c0f waiting on condition [0x000070000553d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #209 daemon prio=5 os_prio=31 tid=0x00007f8d2f9ba000 nid=0x860f waiting on condition [0x000070000543a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007b2423c60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-46" #208 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b9800 nid=0x8a17 in Object.wait() [0x0000700005337000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256b108> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-45" #207 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8800 nid=0xa01b in Object.wait() [0x0000700005234000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b25de4c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #206 daemon prio=5 os_prio=31 tid=0x00007f8d30160000 nid=0x8217 in Object.wait() [0x0000700005131000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256e460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-44" #204 daemon prio=5 os_prio=31 tid=0x00007f8d3015f800 nid=0xb00b in Object.wait() [0x000070000502e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256f060> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-43" #202 daemon prio=5 os_prio=31 tid=0x00007f8d300a4800 nid=0xae0b in Object.wait() [0x0000700004f2b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256c730> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-42" #201 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b8000 nid=0xac0b in Object.wait() [0x0000700004e28000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256cf68> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-41" #200 daemon prio=5 os_prio=31 tid=0x00007f8d2f9b7000 nid=0x7c0f in Object.wait() [0x0000700004d25000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007b256d798> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Delegation Token Canceler" #179 daemon prio=5 os_prio=31 tid=0x00007f8d300ae800 nid=0x680f waiting on condition [0x0000700004919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d3d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer$DelegationTokenCancelThread.run(DelegationTokenRenewer.java:345) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@41f61dd3" #198 daemon prio=5 os_prio=31 tid=0x00007f8d2f800000 nid=0x760f waiting on condition [0x0000700004816000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #197 daemon prio=5 os_prio=31 tid=0x00007f8d2f044800 nid=0xaa13 waiting on condition [0x0000700004713000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #196 daemon prio=5 os_prio=31 tid=0x00007f8d2f044000 nid=0xca07 waiting on condition [0x00007000032d7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #195 daemon prio=5 os_prio=31 tid=0x00007f8d2f043000 nid=0xde0f waiting on condition [0x00007000030d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c29380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #192 daemon prio=5 os_prio=31 tid=0x00007f8d2f134000 nid=0xdc07 waiting on condition [0x0000700002ecb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #191 daemon prio=5 os_prio=31 tid=0x00007f8d2f133800 nid=0xda07 waiting on condition [0x0000700002dc8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #190 daemon prio=5 os_prio=31 tid=0x00007f8d2f96a800 nid=0xc607 waiting on condition [0x0000700002cc5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #189 daemon prio=5 os_prio=31 tid=0x00007f8d3009c000 nid=0xd80f waiting on condition [0x0000700002bc2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-161,5,FailOnTimeoutGroup]" #188 daemon prio=5 os_prio=31 tid=0x00007f8d3009b000 nid=0xd60f waiting on condition [0x0000700002abf000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RM StateStore dispatcher" #186 daemon prio=5 os_prio=31 tid=0x00007f8d2e9d5000 nid=0xd407 waiting on condition [0x00007000029bc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740c67758> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #185 daemon prio=5 os_prio=31 tid=0x00007f8d2b39d000 nid=0xd207 waiting on condition [0x00007000028b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007408d4eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-40" #184 daemon prio=5 os_prio=31 tid=0x00007f8d2b227800 nid=0xd007 in Object.wait() [0x00007000026b3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c68240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-39" #183 daemon prio=5 os_prio=31 tid=0x00007f8d2c7f4800 nid=0xce07 in Object.wait() [0x00007000025b0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c686e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #182 daemon prio=5 os_prio=31 tid=0x00007f8d2b28e800 nid=0xcc07 in Object.wait() [0x00007000024ad000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba5f88> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-38" #180 daemon prio=5 os_prio=31 tid=0x00007f8d2c338800 nid=0xc007 in Object.wait() [0x00007000022a7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740ba6618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-37" #178 daemon prio=5 os_prio=31 tid=0x00007f8d2c3c1800 nid=0xc207 in Object.wait() [0x0000700001b92000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbe460> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-36" #177 daemon prio=5 os_prio=31 tid=0x00007f8d2c5bd800 nid=0xbd07 in Object.wait() [0x0000700001a8f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbeb50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-35" #176 daemon prio=5 os_prio=31 tid=0x00007f8d2c426800 nid=0x3e0b in Object.wait() [0x000070000198c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740cbf360> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-34" #158 daemon prio=5 os_prio=31 tid=0x00007f8d2b227000 nid=0xc703 in Object.wait() [0x0000700004c22000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740c9a998> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-33" #155 daemon prio=5 os_prio=31 tid=0x00007f8d2b0f7800 nid=0xc303 in Object.wait() [0x0000700004a1c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740caabc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.util.JvmPauseMonitor$Monitor@1a795765" #151 daemon prio=5 os_prio=31 tid=0x00007f8d2c0a6000 nid=0xbb03 waiting on condition [0x0000700004610000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #150 daemon prio=5 os_prio=31 tid=0x00007f8d2ea32800 nid=0xb903 waiting on condition [0x000070000450d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ActivitiesManager thread." #149 daemon prio=5 os_prio=31 tid=0x00007f8d2c2de800 nid=0xb703 waiting on condition [0x000070000440a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager$1.run(ActivitiesManager.java:144) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AttributeNodeLabelsManager dispatcher" #148 daemon prio=5 os_prio=31 tid=0x00007f8d2c3ca800 nid=0xb50b waiting on condition [0x0000700004307000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d043f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "NodeLabelManager dispatcher" #145 daemon prio=5 os_prio=31 tid=0x00007f8d2e996000 nid=0xb30b waiting on condition [0x0000700004204000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740d061e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.AsyncDispatcher$1.run(AsyncDispatcher.java:118) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #142 daemon prio=5 os_prio=31 tid=0x00007f8d2e995800 nid=0xb103 waiting on condition [0x0000700004101000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #141 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e7000 nid=0x6117 waiting on condition [0x0000700003ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #140 daemon prio=5 os_prio=31 tid=0x00007f8d2ebf6800 nid=0x9d0b waiting on condition [0x0000700003efb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Ping Checker" #139 daemon prio=5 os_prio=31 tid=0x00007f8d2d8e6000 nid=0x650b waiting on condition [0x0000700003df8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.yarn.util.AbstractLivelinessMonitor$PingChecker.run(AbstractLivelinessMonitor.java:154) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Thread[Thread-115,5,FailOnTimeoutGroup]" #138 daemon prio=5 os_prio=31 tid=0x00007f8d2ea31000 nid=0x630b waiting on condition [0x0000700003cf5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover.run(AbstractDelegationTokenSecretManager.java:698) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "AsyncDispatcher event handler" #136 daemon prio=5 os_prio=31 tid=0x00007f8d3015e800 nid=0x730b waiting on condition [0x0000700003bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740cab540> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.yarn.event.DrainDispatcher$1.run(DrainDispatcher.java:71) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-29" #135 daemon prio=5 os_prio=31 tid=0x00007f8d2b1eb000 nid=0x990b in Object.wait() [0x0000700003aef000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d532f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Node Removal Timer" #134 daemon prio=5 os_prio=31 tid=0x00007f8d2db8f800 nid=0x9513 in Object.wait() [0x00007000039ec000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91488> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-28" #132 daemon prio=5 os_prio=31 tid=0x00007f8d2d90f800 nid=0xa70b in Object.wait() [0x00007000038e9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d99c60> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-27" #131 daemon prio=5 os_prio=31 tid=0x00007f8d2d85b000 nid=0xa50b in Object.wait() [0x00007000037e6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d91b18> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-26" #130 daemon prio=5 os_prio=31 tid=0x00007f8d2d91a800 nid=0xa30b in Object.wait() [0x00007000036e3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e088> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-25" #129 daemon prio=5 os_prio=31 tid=0x00007f8d2d894800 nid=0x910b in Object.wait() [0x00007000034dd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9a1e0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-24" #110 daemon prio=5 os_prio=31 tid=0x00007f8d2d879000 nid=0x870b in Object.wait() [0x00007000023aa000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d92208> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer for 'ResourceManager' metrics system" #107 daemon prio=5 os_prio=31 tid=0x00007f8d2e99a000 nid=0x830b in Object.wait() [0x00007000020a1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9aa30> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-23" #106 daemon prio=5 os_prio=31 tid=0x00007f8d2b38f800 nid=0x6d0b in Object.wait() [0x0000700001e9b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d7e898> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-19" #88 daemon prio=5 os_prio=31 tid=0x00007f8d2ea02800 nid=0x7d07 in Object.wait() [0x00007000035e0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740d9af58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-18" #85 daemon prio=5 os_prio=31 tid=0x00007f8d2b28b000 nid=0x7907 in Object.wait() [0x00007000033da000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da5900> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-14" #67 daemon prio=5 os_prio=31 tid=0x00007f8d2ec87800 nid=0x9b07 in Object.wait() [0x00007000021a4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740da85b8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-13" #63 daemon prio=5 os_prio=31 tid=0x00007f8d2c43a800 nid=0x8c07 in Object.wait() [0x0000700001d98000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e2d760> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-9" #45 daemon prio=5 os_prio=31 tid=0x00007f8d2c487800 nid=0x9303 in Object.wait() [0x00007000031d4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740e1fb08> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-8" #42 daemon prio=5 os_prio=31 tid=0x00007f8d2d96d000 nid=0x8f03 in Object.wait() [0x0000700002fce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740b41e10> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #31 daemon prio=5 os_prio=31 tid=0x00007f8d2b243800 nid=0x7f03 in Object.wait() [0x00007000027b6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000007408fa1e0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3783) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Timer-4" #22 daemon prio=5 os_prio=31 tid=0x00007f8d2b28d800 nid=0x6f03 in Object.wait() [0x0000700001f9e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007400149f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Timer-3" #18 daemon prio=5 os_prio=31 tid=0x00007f8d2ea86000 nid=0x6903 in Object.wait() [0x0000700001c95000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000740014bc0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "surefire-forkedjvm-ping-30s" #11 daemon prio=5 os_prio=31 tid=0x00007f8d2d842800 nid=0x5903 waiting on condition [0x000070000175b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000740014f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=31 tid=0x00007f8d2d025000 nid=0x5703 runnable [0x0000700001658000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:255) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x0000000740022400> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:115) at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:390) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f8d2c009800 nid=0x5303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f8d2c014000 nid=0x5103 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f8d2c013000 nid=0x4f03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f8d2c012800 nid=0x4d03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f8d2c010800 nid=0x4b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f8d2d848000 nid=0x3a0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f8d2e800000 nid=0x3803 in Object.wait() [0x0000700000d3a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000740022750> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f8d2c00f800 nid=0x3603 in Object.wait() [0x0000700000c37000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x0000000740022920> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007f8d2d802000 nid=0x1703 waiting for monitor entry [0x0000700000218000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:217) - waiting to lock <0x000000079b305680> (a java.lang.Object) at org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.tearDown(TestRMRestart.java:174) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007f8d2d811800 nid=0x3403 runnable "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f8d2e004800 nid=0x2403 runnable "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002000 nid=0x2603 runnable "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f8d2d002800 nid=0x2803 runnable "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f8d2c006800 nid=0x2a03 runnable "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f8d2c00d000 nid=0x2c03 runnable "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f8d2b801800 nid=0x2e03 runnable "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f8d2b802000 nid=0x3003 runnable "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f8d2d003000 nid=0x3203 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007f8d2c037000 nid=0x5503 waiting on condition JNI global references: 312