[GEODE-3407] JMX and reconnect may deadlock on CacheFactory.getAnyInstance

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 1.3.0
• Component/s: jmx, membership
• Assignee: Kirk Lund
• Reporter: Kirk Lund
• Created: 07/Aug/17 22:42
• Updated: 20/Apr/18 19:53
• Resolved: 09/Aug/17 17:14

Description

"CloserThread":
	at org.apache.geode.distributed.internal.InternalDistributedSystem.tryReconnect(InternalDistributedSystem.java:2628)
	- waiting to lock <0x0000000244fc5c18> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
	- locked <0x0000000247556560> (a java.lang.Class for org.apache.geode.cache.CacheFactory)
	at org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1055)
	at org.apache.geode.distributed.internal.DistributionManager$MyListener.membershipFailure(DistributionManager.java:4822)
	at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager.uncleanShutdown(JGroupMembershipManager.java:2732)
	at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager$Puller.channelClosing(JGroupMembershipManager.java:1212)
	at org.jgroups.JChannel$CloserThread.run(JChannel.java:1617)

"ReconnectThread":
	at org.apache.geode.management.internal.LocalManager.unMarkForFederation(LocalManager.java:243)
	- waiting to lock <0x0000000351819a00> (a java.lang.Object)
	at org.apache.geode.management.internal.LocalManager.cleanUpResources(LocalManager.java:284)
	at org.apache.geode.management.internal.LocalManager.stopManager(LocalManager.java:441)
	at org.apache.geode.management.internal.SystemManagementService.close(SystemManagementService.java:261)
	- locked <0x000000024cfdfe38> (a java.util.HashMap)
	at org.apache.geode.management.internal.beans.ManagementAdapter.handleCacheRemoval(ManagementAdapter.java:775)
	at org.apache.geode.management.internal.beans.ManagementListener.handleEvent(ManagementListener.java:115)
	at org.apache.geode.distributed.internal.InternalDistributedSystem.notifyResourceEventListeners(InternalDistributedSystem.java:2249)
	at org.apache.geode.distributed.internal.InternalDistributedSystem.handleResourceEvent(InternalDistributedSystem.java:505)
	at org.apache.geode.internal.cache.GemFireCacheImpl.close(GemFireCacheImpl.java:1826)
	- locked <0x0000000244fc5c18> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
	at org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1389)
	- locked <0x0000000244fc5c18> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
	at org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1058)
	at org.apache.geode.distributed.internal.DistributionManager$MyListener.membershipFailure(DistributionManager.java:4822)
	at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager.uncleanShutdown(JGroupMembershipManager.java:2732)
	at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager$Puller.channelClosing(JGroupMembershipManager.java:1212)
	at org.jgroups.JChannel$CloserThread.run(JChannel.java:1617)

"Management Task":
	at org.apache.geode.cache.CacheFactory.getAnyInstance(CacheFactory.java:293)
	- waiting to lock <0x0000000247556560> (a java.lang.Class for org.apache.geode.cache.CacheFactory)
	at org.apache.geode.internal.cache.tier.InternalBridgeMembership.getClientQueueSizes(InternalBridgeMembership.java:358)
	at org.apache.geode.management.internal.beans.CacheServerBridge.getNumSubscriptions(CacheServerBridge.java:666)
	at org.apache.geode.management.internal.beans.CacheServerMBean.getNumSubscriptions(CacheServerMBean.java:288)
	at sun.reflect.GeneratedMethodAccessor300.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.geode.management.internal.FederationComponent.refreshObjectState(FederationComponent.java:175)
	at org.apache.geode.management.internal.LocalManager$ManagementTask.run(LocalManager.java:376)
	- locked <0x0000000351819a00> (a java.lang.Object)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at org.apache.geode.management.internal.LocalManager$1$1.run(LocalManager.java:121)
	at java.lang.Thread.run(Thread.java:745)

Issue Links

• links to: GitHub Pull Request #697