Blocker Description
Captured output from jstack:
Found one Java-level deadlock: ============================= "cron4j::scheduler[b825ad9442919b58656dd08c00000143930e6d2e3cc88920]::executor[b825ad9442919b5804431808000001439dfe86287be633f2]": waiting for ownable synchronizer 0x0000000663057dd0, (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync), which is held by "TaskScheduler-0" "TaskScheduler-0": waiting to lock monitor 0x00007fe0d8c98e58 (object 0x0000000663021fc8, a org.apache.aurora.scheduler.state.SchedulerCoreImpl), which is held by "cron4j::scheduler[b825ad9442919b58656dd08c00000143930e6d2e3cc88920]::executor[b825ad9442919b58704e3eda000001439ded211c188378cc]" "cron4j::scheduler[b825ad9442919b58656dd08c00000143930e6d2e3cc88920]::executor[b825ad9442919b58704e3eda000001439ded211c188378cc]": waiting to lock monitor 0x00007fe034858768 (object 0x0000000662fffab8, a org.apache.aurora.scheduler.storage.log.LogStorage$$EnhancerByGuice$$8c0089), which is held by "TaskScheduler-0" Java stack information for the threads listed above: =================================================== "cron4j::scheduler[b825ad9442919b58656dd08c00000143930e6d2e3cc88920]::executor[b825ad9442919b5804431808000001439dfe86287be633f2]": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000663057dd0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:964) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:731) at org.apache.aurora.scheduler.storage.ReadWriteLockManager.readLock(ReadWriteLockManager.java:75) at org.apache.aurora.scheduler.storage.mem.MemStorage.consistentRead(MemStorage.java:114) at org.apache.aurora.scheduler.storage.mem.MemStorage$$EnhancerByGuice$$aa99778b.CGLIB$consistentRead$1(<generated>) at org.apache.aurora.scheduler.storage.mem.MemStorage$$EnhancerByGuice$$aa99778b$$FastClassByGuice$$d840358f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52) at org.apache.aurora.scheduler.storage.mem.MemStorage$$EnhancerByGuice$$aa99778b.consistentRead(<generated>) at org.apache.aurora.scheduler.storage.ForwardingStore.consistentRead(ForwardingStore.java:92) at org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.consistentRead(CallOrderEnforcingStorage.java:116) at org.apache.aurora.scheduler.storage.Storage$Util.consistentFetchTasks(Storage.java:279) at org.apache.aurora.scheduler.state.CronJobManager.cronTriggered(CronJobManager.java:279) at org.apache.aurora.scheduler.state.CronJobManager$6.run(CronJobManager.java:401) at it.sauronsoftware.cron4j.RunnableTask.execute(Unknown Source) at it.sauronsoftware.cron4j.TaskExecutor$Runner.run(Unknown Source) at java.lang.Thread.run(Thread.java:744) "TaskScheduler-0": at org.apache.aurora.scheduler.state.SchedulerCoreImpl.preemptTask(SchedulerCoreImpl.java:321) - waiting to lock <0x0000000663021fc8> (a org.apache.aurora.scheduler.state.SchedulerCoreImpl) at org.apache.aurora.scheduler.async.Preemptor$PreemptorImpl.findPreemptionSlotFor(Preemptor.java:343) - locked <0x0000000662f9ba98> (a org.apache.aurora.scheduler.async.Preemptor$PreemptorImpl) at org.apache.aurora.scheduler.async.TaskScheduler$TaskSchedulerImpl.maybePreemptFor(TaskScheduler.java:205) at org.apache.aurora.scheduler.async.TaskScheduler$TaskSchedulerImpl.access$600(TaskScheduler.java:91) at org.apache.aurora.scheduler.async.TaskScheduler$TaskSchedulerImpl$2.apply(TaskScheduler.java:173) at org.apache.aurora.scheduler.async.TaskScheduler$TaskSchedulerImpl$2.apply(TaskScheduler.java:161) at org.apache.aurora.scheduler.storage.log.LogStorage$7.apply(LogStorage.java:492) at org.apache.aurora.scheduler.storage.mem.MemStorage.write(MemStorage.java:146) at org.apache.aurora.scheduler.storage.mem.MemStorage$$EnhancerByGuice$$aa99778b.CGLIB$write$0(<generated>) at org.apache.aurora.scheduler.storage.mem.MemStorage$$EnhancerByGuice$$aa99778b$$FastClassByGuice$$d840358f.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52) at org.apache.aurora.scheduler.storage.mem.MemStorage$$EnhancerByGuice$$aa99778b.write(<generated>) at org.apache.aurora.scheduler.storage.ForwardingStore.write(ForwardingStore.java:105) at org.apache.aurora.scheduler.storage.log.LogStorage.write(LogStorage.java:490) - locked <0x0000000662fffab8> (a org.apache.aurora.scheduler.storage.log.LogStorage$$EnhancerByGuice$$8c0089) at org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.write(CallOrderEnforcingStorage.java:131) "cron4j::scheduler[b825ad9442919b58656dd08c00000143930e6d2e3cc88920]::executor[b825ad9442919b58704e3eda000001439ded211c188378cc]" daemon prio=10 tid=0x00007fe02c184000 nid=0xc40a waiting for monitor entry [0x0000000055f9f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.aurora.scheduler.storage.log.LogStorage.write(LogStorage.java:474) - waiting to lock <0x0000000662fffab8> (a org.apache.aurora.scheduler.storage.log.LogStorage$$EnhancerByGuice$$8c0089) at org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.write(CallOrderEnforcingStorage.java:131) at org.apache.aurora.scheduler.state.SideEffectStorage.write(SideEffectStorage.java:88) at org.apache.aurora.scheduler.state.StateManagerImpl.changeState(StateManagerImpl.java:244) at org.apache.aurora.scheduler.state.StateManagerImpl.changeState(StateManagerImpl.java:203) at org.apache.aurora.scheduler.state.SchedulerCoreImpl.killTasks(SchedulerCoreImpl.java:279) - locked <0x0000000663021fc8> (a org.apache.aurora.scheduler.state.SchedulerCoreImpl) at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at com.google.inject.internal.DelegatingInvocationHandler.invoke(DelegatingInvocationHandler.java:37) at com.sun.proxy.$Proxy65.killTasks(Unknown Source) at org.apache.aurora.scheduler.state.CronJobManager.cronTriggered(CronJobManager.java:290) at org.apache.aurora.scheduler.state.CronJobManager$6.run(CronJobManager.java:401) at it.sauronsoftware.cron4j.RunnableTask.execute(Unknown Source) at it.sauronsoftware.cron4j.TaskExecutor$Runner.run(Unknown Source) at java.lang.Thread.run(Thread.java:744)