[GEODE-5472] PersistentPartitionedRegionRegressionTest recoversAfterBucketCreationCrashes hangs

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 1.7.0
• Labels: pull-request-available swat
• Assignee: Jinmei Liao
• Reporter: Jinmei Liao
• Created: 24/Jul/18 17:21
• Updated: 03/Oct/18 21:37
• Resolved: 26/Jul/18 19:03

Description

https://concourse.apachegeode-ci.info/teams/main/pipelines/develop/jobs/DistributedTest/builds/138

 

Pooled Waiting Message Processor 1 is in deadlock with RMI TCP Connection(1)-172.17.0.5

Pooled Waiting Message Processor 1 - priority:10 - threadId:0x00007ff740004800 - nativeId:0x498 - state:BLOCKED
stackTrace:
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.geode.internal.cache.GemFireCacheImpl.removeRoot(GemFireCacheImpl.java:3604)

• waiting to lock <0x00000000e0822280> (a java.util.HashMap)
  at org.apache.geode.internal.cache.LocalRegion.basicDestroyRegion(LocalRegion.java:6288)
  at org.apache.geode.internal.cache.DistributedRegion.basicDestroyRegion(DistributedRegion.java:1730)
  at org.apache.geode.internal.cache.LocalRegion.basicDestroyRegion(LocalRegion.java:6211)
  at org.apache.geode.internal.cache.LocalRegion.localDestroyRegion(LocalRegion.java:2229)
  at org.apache.geode.internal.cache.AbstractRegion.localDestroyRegion(AbstractRegion.java:424)
  at org.apache.geode.management.internal.ManagementResourceRepo.destroyLocalMonitoringRegion(ManagementResourceRepo.java:73)
  at org.apache.geode.management.internal.LocalManager.cleanUpResources(LocalManager.java:279)
  at org.apache.geode.management.internal.LocalManager.stopManager(LocalManager.java:413)
  at org.apache.geode.management.internal.SystemManagementService.close(SystemManagementService.java:240)
• locked <0x00000000e03dcfe0> (a java.util.HashMap)
  at org.apache.geode.management.internal.beans.ManagementAdapter.handleCacheRemoval(ManagementAdapter.java:737)
  at org.apache.geode.management.internal.beans.ManagementListener.handleEvent(ManagementListener.java:122)
  at org.apache.geode.distributed.internal.InternalDistributedSystem.notifyResourceEventListeners(InternalDistributedSystem.java:2201)
  at org.apache.geode.distributed.internal.InternalDistributedSystem.handleResourceEvent(InternalDistributedSystem.java:590)
  at org.apache.geode.internal.cache.GemFireCacheImpl.close(GemFireCacheImpl.java:2147)
• locked <0x00000000e03dd028> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
  at org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1371)
• locked <0x00000000e03dd028> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
  at org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1021)
  at org.apache.geode.test.dunit.Disconnect.disconnectFromDS(Disconnect.java:43)
  at org.apache.geode.internal.cache.partitioned.PersistentPartitionedRegionRegressionTest$1.beforeSendMessage(PersistentPartitionedRegionRegressionTest.java:288)
  at org.apache.geode.distributed.internal.ClusterDistributionManager.putOutgoing(ClusterDistributionManager.java:1720)
  at org.apache.geode.internal.cache.partitioned.ManageBucketMessage$ManageBucketReplyMessage.sendAcceptance(ManageBucketMessage.java:278)
  at org.apache.geode.internal.cache.partitioned.ManageBucketMessage.operateOnPartitionedRegion(ManageBucketMessage.java:152)
  at org.apache.geode.internal.cache.partitioned.PartitionMessage.process(PartitionMessage.java:331)
  at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:378)
  at org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:444)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at org.apache.geode.distributed.internal.ClusterDistributionManager.runUntilShutdown(ClusterDistributionManager.java:1121)
  at org.apache.geode.distributed.internal.ClusterDistributionManager.access$000(ClusterDistributionManager.java:109)
  at org.apache.geode.distributed.internal.ClusterDistributionManager$6$1.run(ClusterDistributionManager.java:865)
  at java.lang.Thread.run(Thread.java:748)
  Locked ownable synchronizers:
  - <0x00000000e08286b0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
  - <0x00000000e082be50> (a java.util.concurrent.ThreadPoolExecutor$Worker)

RMI TCP Connection(1)-172.17.0.5 - priority:5 - threadId:0x00007ff7a4001800 - nativeId:0x18d - state:WAITING
stackTrace:
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)

• parking to wait for <0x00000000e08286b0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
  at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967)
  at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283)
  at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727)
  at org.apache.geode.management.internal.beans.ManagementListener.handleEvent(ManagementListener.java:113)
  at org.apache.geode.distributed.internal.InternalDistributedSystem.notifyResourceEventListeners(InternalDistributedSystem.java:2201)
  at org.apache.geode.distributed.internal.InternalDistributedSystem.handleResourceEvent(InternalDistributedSystem.java:590)
  at org.apache.geode.internal.cache.DiskStoreFactoryImpl.create(DiskStoreFactoryImpl.java:143)
• locked <0x00000000e0822120> (a org.apache.geode.internal.cache.GemFireCacheImpl)
  at org.apache.geode.internal.cache.GemFireCacheImpl.getOrCreateDefaultDiskStore(GemFireCacheImpl.java:2591)
• locked <0x00000000e0822120> (a org.apache.geode.internal.cache.GemFireCacheImpl)
  at org.apache.geode.internal.cache.LocalRegion.findDiskStore(LocalRegion.java:7605)
  at org.apache.geode.internal.cache.PartitionedRegion.findDiskStore(PartitionedRegion.java:9053)
  at org.apache.geode.internal.cache.LocalRegion.<init>(LocalRegion.java:637)
  at org.apache.geode.internal.cache.PartitionedRegion.<init>(PartitionedRegion.java:740)
  at org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:3040)
• locked <0x00000000e0822280> (a java.util.HashMap)
  at org.apache.geode.internal.cache.GemFireCacheImpl.basicCreateRegion(GemFireCacheImpl.java:2981)
  at org.apache.geode.internal.cache.GemFireCacheImpl.createRegion(GemFireCacheImpl.java:2969)
  at org.apache.geode.cache.RegionFactory.create(RegionFactory.java:756)
  at org.apache.geode.internal.cache.partitioned.PersistentPartitionedRegionRegressionTest.createPartitionedRegion(PersistentPartitionedRegionRegressionTest.java:555)
  at org.apache.geode.internal.cache.partitioned.PersistentPartitionedRegionRegressionTest.lambda$recoversAfterBucketCreationCrashes$bb17a952$6(PersistentPartitionedRegionRegressionTest.java:306)
  at org.apache.geode.internal.cache.partitioned.PersistentPartitionedRegionRegressionTest$$Lambda$89/1886884226.run(Unknown Source)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:498)
  at hydra.MethExecutor.executeObject(MethExecutor.java:244)
  at org.apache.geode.test.dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:70)
  at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source)
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:498)
  at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:357)
  at sun.rmi.transport.Transport$1.run(Transport.java:200)
  at sun.rmi.transport.Transport$1.run(Transport.java:197)
  at java.security.AccessController.doPrivileged(Native Method)
  at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
  at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:573)
  at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:835)
  at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:688)
  at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$7/1029844808.run(Unknown Source)
  at java.security.AccessController.doPrivileged(Native Method)
  at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:687)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)
  Locked ownable synchronizers:
  - <0x00000000e03ebb68> (a java.util.concurrent.ThreadPoolExecutor$Worker)

Issue Links

• links to: GitHub Pull Request #2186