Minor Description
Looks like LocalSchedulerMain waits forever on an attempt to cancel its zookeeper leadership
"qtp516227519-124" prio=10 tid=0x00007f65a8e16000 nid=0x4c50 waiting for monitor entry [0x00007f648faf5000] java.lang.Thread.State: BLOCKED (on object monitor) at com.twitter.common.zookeeper.Group$ActiveMembership.cancel(Group.java:365) - waiting to lock <0x000000060f86d9a8> (a com.twitter.common.zookeeper.Group$ActiveMembership) at com.twitter.common.zookeeper.ServerSetImpl$MemberStatus.leave(ServerSetImpl.java:255) - locked <0x000000060f86ced0> (a com.twitter.common.zookeeper.ServerSetImpl$MemberStatus) at com.twitter.common.zookeeper.ServerSetImpl$2.leave(ServerSetImpl.java:199) at com.twitter.common.zookeeper.SingletonService$1$1.leave(SingletonService.java:185) - locked <0x000000060f86ce98> (a com.twitter.common.zookeeper.SingletonService$1$1) at org.apache.aurora.scheduler.SchedulerLifecycle$8.execute(SchedulerLifecycle.java:367) at org.apache.aurora.scheduler.SchedulerLifecycle$8.execute(SchedulerLifecycle.java:352) at com.twitter.common.base.Closures$4.execute(Closures.java:120) at com.twitter.common.base.Closures$3.execute(Closures.java:98) at com.twitter.common.util.StateMachine.transition(StateMachine.java:191) at org.apache.aurora.scheduler.SchedulerLifecycle$4.execute(SchedulerLifecycle.java:232) at com.twitter.common.application.ShutdownRegistry$ShutdownRegistryImpl.execute(ShutdownRegistry.java:88) - locked <0x000000060f86c6c0> (a com.twitter.common.application.ShutdownRegistry$ShutdownRegistryImpl) at com.twitter.common.application.Lifecycle.shutdown(Lifecycle.java:92) - locked <0x000000060f86c6b0> (a java.lang.Object) at org.apache.aurora.scheduler.storage.backup.BackupModule$LifecycleHook.execute(BackupModule.java:117) at org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl$PendingRecovery$1.execute(Recovery.java:198) at org.apache.aurora.scheduler.storage.Storage$MutateWork$NoResult.apply(Storage.java:132) at org.apache.aurora.scheduler.storage.Storage$MutateWork$NoResult$Quiet.apply(Storage.java:149) at org.apache.aurora.scheduler.storage.mem.MemStorage.doWork(MemStorage.java:175) at org.apache.aurora.scheduler.storage.mem.MemStorage.access$400(MemStorage.java:59) at org.apache.aurora.scheduler.storage.mem.MemStorage$4.apply(MemStorage.java:200) at org.apache.aurora.scheduler.storage.mem.MemStorage$4.apply(MemStorage.java:197) at org.apache.aurora.scheduler.storage.db.DbStorage.write(DbStorage.java:150) at org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366.CGLIB$write$4(<generated>) at org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366$$FastClassByGuice$$9e51bc15.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at org.mybatis.guice.transactional.TransactionalMethodInterceptor.invoke(TransactionalMethodInterceptor.java:101) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52) at org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366.write(<generated>) at org.apache.aurora.scheduler.storage.mem.MemStorage.write(MemStorage.java:197) at org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl$PendingRecovery.commit(Recovery.java:193) at org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl.commit(Recovery.java:182) at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.commitRecovery(SchedulerThriftInterface.java:1009) at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84.CGLIB$commitRecovery$7(<generated>) at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84$$FastClassByGuice$$455b50c2.invoke(<generated>) at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at org.apache.aurora.scheduler.thrift.aop.ServerInfoInterceptor.invoke(ServerInfoInterceptor.java:32) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at org.apache.aurora.scheduler.thrift.aop.ThriftStatsExporterInterceptor.invoke(ThriftStatsExporterInterceptor.java:47) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at org.apache.aurora.scheduler.thrift.aop.FeatureToggleInterceptor.invoke(FeatureToggleInterceptor.java:38) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at org.apache.aurora.scheduler.thrift.aop.UserCapabilityInterceptor.invoke(UserCapabilityInterceptor.java:92) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at org.apache.aurora.scheduler.thrift.aop.LoggingInterceptor.invoke(LoggingInterceptor.java:87) at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72) at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52) at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84.commitRecovery(<generated>) at org.apache.aurora.gen.AuroraAdmin$Processor$commitRecovery.getResult(AuroraAdmin.java:1296) at org.apache.aurora.gen.AuroraAdmin$Processor$commitRecovery.getResult(AuroraAdmin.java:1281) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.thrift.server.TServlet.doPost(TServlet.java:83) at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263) at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178) at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:82) at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:294) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118) at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1288) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) at org.eclipse.jetty.server.Server.handle(Server.java:369) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486) at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:944) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1005) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:667) at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745)
"main-EventThread" daemon prio=10 tid=0x00007f65a8e52000 nid=0x4c56 waiting on condition [0x00007f648f4f3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000076ed20ad8> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:236) at com.twitter.common.zookeeper.ZooKeeperClient.get(ZooKeeperClient.java:398) - locked <0x000000060f86d060> (a com.twitter.common.zookeeper.ZooKeeperClient) at com.twitter.common.zookeeper.ZooKeeperClient.get(ZooKeeperClient.java:332) - locked <0x000000060f86d060> (a com.twitter.common.zookeeper.ZooKeeperClient) at com.twitter.common.zookeeper.Group$ActiveMembership.join(Group.java:424) - locked <0x000000060f86d9a8> (a com.twitter.common.zookeeper.Group$ActiveMembership) at com.twitter.common.zookeeper.Group$ActiveMembership$4.get(Group.java:445) at com.twitter.common.zookeeper.Group$ActiveMembership$4.get(Group.java:442) at com.twitter.common.util.BackoffHelper$1.get(BackoffHelper.java:109) at com.twitter.common.util.BackoffHelper$1.get(BackoffHelper.java:107) at com.twitter.common.util.BackoffHelper.doUntilResult(BackoffHelper.java:127) at com.twitter.common.util.BackoffHelper.doUntilSuccess(BackoffHelper.java:107) at com.twitter.common.zookeeper.Group$ActiveMembership.tryJoin(Group.java:467) - locked <0x000000060f86d9a8> (a com.twitter.common.zookeeper.Group$ActiveMembership) at com.twitter.common.zookeeper.Group$ActiveMembership.access$800(Group.java:316) at com.twitter.common.zookeeper.Group$ActiveMembership$3.process(Group.java:433) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:507)
Attachments
Issue Links
- duplicates
-
AURORA-327 Aurora shutdown blocks when attempting to abdicate leadership and zk ensemble is down
-
- Resolved
-