[GEODE-1999] offheap memory leak could occur in GatewaySenderQueue

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 1.0.0-incubating
• Component/s: offheap
• Assignee: Eric Shu
• Reporter: Eric Shu
• Created: 12/Oct/16 23:04
• Updated: 25/Oct/16 07:17
• Resolved: 13/Oct/16 17:55

Description

When GatewaySenderEventImpl is removing from the sender queue, it is put into an EntryEventImpl. The offheap reference will be removed late once the basicDestory call is done.

protected void basicDestroy(final EntryEventImpl event,
final boolean cacheWrite, Object expectedOldValue)
throws EntryNotFoundException, CacheWriterException, TimeoutException

{ super.basicDestroy(event, cacheWrite, expectedOldValue); GatewaySenderEventImpl.release(event.getRawOldValue()); }

The following exception is thrown during the basicDestroy call, and the GatewaySenderEventImpl removed from the queue in the basicDestroy failed to remove offheap reference.

[info 2016/10/11 11:07:38.312 PDT bridgegemfire_1_1_elmax_2881 <ServerConnection on port 29381 Thread 1> tid=0x76] org.apache.geode.distributed.DistributedSystemDisconnectedException: DistributedSystem is shutting down, caused by org.apache.geode.ForcedDisconnectException: Member isn't responding to heartbeat requests
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.directChannelSend(GMSMembershipManager.java:1701)
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.send(GMSMembershipManager.java:1861)
at org.apache.geode.distributed.internal.DistributionChannel.send(DistributionChannel.java:88)
at org.apache.geode.distributed.internal.DistributionManager.sendOutgoing(DistributionManager.java:3402)
at org.apache.geode.distributed.internal.DistributionManager.sendMessage(DistributionManager.java:3443)
at org.apache.geode.distributed.internal.DistributionManager.putOutgoing(DistributionManager.java:1806)
at org.apache.geode.internal.cache.DistributedCacheOperation.distribute(DistributedCacheOperation.java:509)
at org.apache.geode.internal.cache.DistributedRegion.distributeDestroy(DistributedRegion.java:1779)
at org.apache.geode.internal.cache.DistributedRegion.basicDestroyPart3(DistributedRegion.java:1770)
at org.apache.geode.internal.cache.AbstractRegionMap.destroy(AbstractRegionMap.java:1527)
at org.apache.geode.internal.cache.LocalRegion.mapDestroy(LocalRegion.java:7019)
at org.apache.geode.internal.cache.LocalRegion.mapDestroy(LocalRegion.java:6991)
at org.apache.geode.internal.cache.LocalRegionDataView.destroyExistingEntry(LocalRegionDataView.java:55)
at org.apache.geode.internal.cache.LocalRegion.basicDestroy(LocalRegion.java:6956)
at org.apache.geode.internal.cache.DistributedRegion.basicDestroy(DistributedRegion.java:1738)
at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue$SerialGatewaySenderQueueMetaRegion.basicDestroy(SerialGatewaySenderQueue.java:1298)
at org.apache.geode.internal.cache.LocalRegion.validatedDestroy(LocalRegion.java:1150)
at org.apache.geode.internal.cache.DistributedRegion.validatedDestroy(DistributedRegion.java:1024)
at org.apache.geode.internal.cache.LocalRegion.destroy(LocalRegion.java:1134)
at org.apache.geode.internal.cache.AbstractRegion.destroy(AbstractRegion.java:276)
at org.apache.geode.internal.cache.LocalRegion.remove(LocalRegion.java:9587)
at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue.removeOldEntry(SerialGatewaySenderQueue.java:657)
at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue.putAndGetKey(SerialGatewaySenderQueue.java:279)
at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueue.put(SerialGatewaySenderQueue.java:252)
at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.queuePrimaryEvent(SerialGatewaySenderEventProcessor.java:467)
at org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.enqueueEvent(SerialGatewaySenderEventProcessor.java:436)
at org.apache.geode.internal.cache.wan.AbstractGatewaySender.distribute(AbstractGatewaySender.java:959)
at org.apache.geode.internal.cache.LocalRegion.notifyGatewaySender(LocalRegion.java:6599)
at org.apache.geode.internal.cache.BucketRegion.notifyGatewaySender(BucketRegion.java:673)
at org.apache.geode.internal.cache.LocalRegion.basicPutPart2(LocalRegion.java:6136)
at org.apache.geode.internal.cache.BucketRegion.basicPutPart2(BucketRegion.java:663)
at org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2807)
at org.apache.geode.internal.cache.BucketRegion.virtualPut(BucketRegion.java:504)
at org.apache.geode.internal.cache.PartitionedRegionDataStore.putLocally(PartitionedRegionDataStore.java:1275)
at org.apache.geode.internal.cache.PartitionedRegion.putInBucket(PartitionedRegion.java:2809)
at org.apache.geode.internal.cache.PartitionedRegion.virtualPut(PartitionedRegion.java:1973)
at org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:132)
at org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5930)
at org.apache.geode.internal.cache.LocalRegion.basicBridgePut(LocalRegion.java:5548)
at org.apache.geode.internal.cache.tier.sockets.command.Put65.cmdExecute(Put65.java:367)
at org.apache.geode.internal.cache.tier.sockets.BaseCommand.execute(BaseCommand.java:147)
at org.apache.geode.internal.cache.tier.sockets.ServerConnection.doNormalMsg(ServerConnection.java:783)
at org.apache.geode.internal.cache.tier.sockets.ServerConnection.doOneMessage(ServerConnection.java:913)
at org.apache.geode.internal.cache.tier.sockets.ServerConnection.run(ServerConnection.java:1143)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.geode.internal.cache.tier.sockets.AcceptorImpl$1$1.run(AcceptorImpl.java:546)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.geode.ForcedDisconnectException: Member isn't responding to heartbeat requests
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.forceDisconnect(GMSMembershipManager.java:2502)
at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.forceDisconnect(GMSJoinLeave.java:971)
at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.processRemoveRequest(GMSJoinLeave.java:618)
at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.processMessage(GMSJoinLeave.java:1654)
at org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.receive(JGroupsMessenger.java:1269)
at org.jgroups.JChannel.invokeCallback(JChannel.java:816)
at org.jgroups.JChannel.up(JChannel.java:741)
at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030)
at org.jgroups.protocols.FRAG2.up(FRAG2.java:165)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:390)
at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1070)
at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:785)
at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426)
at org.apache.geode.distributed.internal.membership.gms.messenger.StatRecorder.up(StatRecorder.java:75)
at org.apache.geode.distributed.internal.membership.gms.messenger.AddressManager.up(AddressManager.java:75)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1601)
at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1817)
at org.jgroups.util.DirectExecutor.execute(DirectExecutor.java:10)
at org.jgroups.protocols.TP.handleSingleMessage(TP.java:1729)
at org.jgroups.protocols.TP.receive(TP.java:1654)
at org.apache.geode.distributed.internal.membership.gms.messenger.Transport.receive(Transport.java:162)
at org.jgroups.protocols.UDP$PacketReceiver.run(UDP.java:701)
... 1 more