[IGNITE-12828] Intermittent [Failed to notify direct custom event listener] exception on node shutdown

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 2.8
• Fix Version/s: 2.8.1
• Labels: sbcf
• Assignee: Mikhail Petrov
• Reporter: Alexey Kukushkin
• Created: 22/Mar/20 22:18
• Updated: 09/Nov/21 11:40
• Resolved: 08/Apr/20 15:17

Description

Reproducer:

Run a server node

Run a client node that:

• Creates cache "cache1"
• Deploys a grid service that starts a continuous query against "cache1" in method init()
• Leaves the cluster

Actual result

Intermittent exception in the client node:

[16:54:38,758][SEVERE][disco-notifier-worker-#43%CashFlowCluster_16b67e98563f4cfbac95ae055a00e67f%][GridDiscoveryManager] Failed to notify direct custom event listener: StartRoutineDiscoveryMessage [startReqData=StartRequestData [prjPred=sbt.cashflow.grid.services.cachefactory.ignite.NodeAttributeFilter@63ae71a9, clsName=null, depInfo=null, hnd=CacheContinuousQueryHandler [returnValTrans=o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$1@594bf5b8, cacheName=CALC_REQUESTS, rmtFilter=null, rmtFilterDep=null, internal=false, notifyExisting=false, oldValRequired=true, sync=false, ignoreExpired=true, taskHash=0, skipPrimaryCheck=false, locOnly=false, keepBinary=true, ackBuf=null, cacheId=-1608655250, initTopVer=null, nodeLeft=false, ignoreClsNotFound=false, nodeId=null, routineId=null], bufSize=1, interval=0, autoUnsubscribe=true], keepBinary=true, routineId=021dd2ce-3d8a-41c1-a4d0-b625ea1284f4]
java.lang.NullPointerException
 at org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage.addUpdateCounters(StartRoutineDiscoveryMessage.java:82)
 at org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage.addUpdateCounters(StartRoutineDiscoveryMessage.java:96)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.processStartRequest(GridContinuousProcessor.java:1424)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.access$400(GridContinuousProcessor.java:110)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$2.onCustomEvent(GridContinuousProcessor.java:202)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$2.onCustomEvent(GridContinuousProcessor.java:193)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery0(GridDiscoveryManager.java:722)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.lambda$onDiscovery$0(GridDiscoveryManager.java:601)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2683)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2721)
 at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119)
 at java.lang.Thread.run(Thread.java:745)
[16:54:39,725][SEVERE][disco-notifier-worker-#43%CashFlowCluster_16b67e98563f4cfbac95ae055a00e67f%][GridDiscoveryManager] Failed to notify direct custom event listener: StartRoutineDiscoveryMessage [startReqData=StartRequestData [prjPred=sbt.cashflow.grid.services.cachefactory.ignite.NodeAttributeFilter@7462c96c, clsName=null, depInfo=null, hnd=CacheContinuousQueryHandler [returnValTrans=o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$1@6451dd70, cacheName=DISTRIBUTED_REQUESTS, rmtFilter=null, rmtFilterDep=null, internal=false, notifyExisting=false, oldValRequired=true, sync=false, ignoreExpired=true, taskHash=0, skipPrimaryCheck=false, locOnly=false, keepBinary=true, ackBuf=null, cacheId=1419803136, initTopVer=null, nodeLeft=false, ignoreClsNotFound=false, nodeId=null, routineId=null], bufSize=1, interval=0, autoUnsubscribe=true], keepBinary=true, routineId=1fca5f04-d220-49ac-850a-0d4527e22eef]
java.lang.NullPointerException
 at org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage.addUpdateCounters(StartRoutineDiscoveryMessage.java:82)
 at org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage.addUpdateCounters(StartRoutineDiscoveryMessage.java:96)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.processStartRequest(GridContinuousProcessor.java:1424)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.access$400(GridContinuousProcessor.java:110)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$2.onCustomEvent(GridContinuousProcessor.java:202)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$2.onCustomEvent(GridContinuousProcessor.java:193)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery0(GridDiscoveryManager.java:722)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.lambda$onDiscovery$0(GridDiscoveryManager.java:601)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2683)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2721)
 at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119)
 at java.lang.Thread.run(Thread.java:745)
[16:54:40,809][SEVERE][disco-notifier-worker-#43%CashFlowCluster_16b67e98563f4cfbac95ae055a00e67f%][GridDiscoveryManager] Failed to notify direct custom event listener: StartRoutineDiscoveryMessage [startReqData=StartRequestData [prjPred=sbt.cashflow.grid.services.cachefactory.ignite.NodeAttributeFilter@4a29e4c8, clsName=null, depInfo=null, hnd=CacheContinuousQueryHandler [returnValTrans=o.a.i.i.processors.cache.query.continuous.CacheContinuousQueryHandler$1@28627d48, cacheName=DISTRIBUTED_REQUESTS, rmtFilter=null, rmtFilterDep=null, internal=false, notifyExisting=false, oldValRequired=true, sync=false, ignoreExpired=true, taskHash=0, skipPrimaryCheck=false, locOnly=false, keepBinary=true, ackBuf=null, cacheId=1419803136, initTopVer=null, nodeLeft=false, ignoreClsNotFound=false, nodeId=null, routineId=null], bufSize=1, interval=0, autoUnsubscribe=true], keepBinary=true, routineId=aa0bdf4f-bfdb-4eb3-8d99-6bcb67532704]
java.lang.NullPointerException
 at org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage.addUpdateCounters(StartRoutineDiscoveryMessage.java:82)
 at org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage.addUpdateCounters(StartRoutineDiscoveryMessage.java:96)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.processStartRequest(GridContinuousProcessor.java:1424)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.access$400(GridContinuousProcessor.java:110)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$2.onCustomEvent(GridContinuousProcessor.java:202)
 at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$2.onCustomEvent(GridContinuousProcessor.java:193)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery0(GridDiscoveryManager.java:722)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.lambda$onDiscovery$0(GridDiscoveryManager.java:601)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2683)
 at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2721)
 at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119)
 at java.lang.Thread.run(Thread.java:745)

Issue Links

• relates to: IGNITE-12865 NPE occurs during CQ registration with cache node filter specified.
• links to: GitHub Pull Request #7620