2018-12-11 13:26:43,668 [ERROR] [org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi] [sys-#156%DPL_GRID%DplGridNodeName%] - Failed to send message to remote node [node=TcpDiscoveryNode [id=c4b95513-618b-44b0-9eb2-e1cda5e44c2d, addrs=ArrayList [10.116.214.78], sockAddrs=HashSet [grid596.ca.sbrf.ru/10.116.214.78:47500], discPort=47500, order=140, intOrder=140, lastExchangeTime=1544523778497, loc=false, ver=2.5.1#20181126-sha1:d6ed9169, isClient=false], msg=GridIoMessage [plc=10, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridCacheIdMessage [cacheId=-2100569601]GridCacheQueryRequest [id=2, cacheName=ignite-sys-cache, type=SCAN, fields=false, clause=null, clsName=null, keyValFilter=null, rdc=null, trans=null, pageSize=1024, incBackups=false, cancel=false, incMeta=false, all=false, keepBinary=false, subjId=42d72760-306a-4175-af19-18db697c8b81, taskHash=0, part=-1, topVer=AffinityTopologyVersion [topVer=234, minorTopVer=0], super=]]] org.apache.ignite.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=c4b95513-618b-44b0-9eb2-e1cda5e44c2d, addrs=[grid596.ca.sbrf.ru/10.116.214.78:47100]] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3460) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createNioClient(TcpCommunicationSpi.java:2982) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.reserveClient(TcpCommunicationSpi.java:2863) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:2706) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:2665) at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1643) at org.apache.ignite.internal.managers.communication.GridIoManager.sendToGridTopic(GridIoManager.java:1715) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1177) at org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager.sendRequest(GridCacheDistributedQueryManager.java:816) at org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager.queryDistributed(GridCacheDistributedQueryManager.java:569) at org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager.scanQueryDistributed(GridCacheDistributedQueryManager.java:609) at org.apache.ignite.internal.processors.cache.query.GridCacheQueryAdapter.executeScanQuery(GridCacheQueryAdapter.java:530) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.igniteIterator(GridCacheAdapter.java:3985) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.igniteIterator(GridCacheAdapter.java:3968) at org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.existingEntries(CacheContinuousQueryManager.java:793) at org.apache.ignite.internal.processors.service.GridServiceProcessor$1.run(GridServiceProcessor.java:269) at org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:6782) at org.apache.ignite.internal.processors.closure.GridClosureProcessor$1.body(GridClosureProcessor.java:827) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to address [addr=grid596.ca.sbrf.ru/10.116.214.78:47100, err=Failed to read remote node recovery handshake (connection closed).] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3463) ... 21 common frames omitted Caused by: org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException: Failed to read remote node recovery handshake (connection closed). at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeTcpHandshake(TcpCommunicationSpi.java:3753) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3314) ... 21 common frames omitted Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to address [addr=grid596.ca.sbrf.ru/10.116.214.78:47100, err=Failed to read remote node recovery handshake (connection closed).] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3463) ... 21 common frames omitted Caused by: org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException: Failed to read remote node recovery handshake (connection closed). at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeTcpHandshake(TcpCommunicationSpi.java:3753) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3314) ... 21 common frames omitted Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to address [addr=grid596.ca.sbrf.ru/10.116.214.78:47100, err=Failed to read remote node recovery handshake (connection closed).] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3463) ... 21 common frames omitted Caused by: org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException: Failed to read remote node recovery handshake (connection closed). at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeTcpHandshake(TcpCommunicationSpi.java:3753) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3314) ... 21 common frames omitted Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to address [addr=grid596.ca.sbrf.ru/10.116.214.78:47100, err=Failed to read remote node recovery handshake (connection closed).] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3463) ... 21 common frames omitted Caused by: org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException: Failed to read remote node recovery handshake (connection closed). at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeTcpHandshake(TcpCommunicationSpi.java:3753) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3314) ... 21 common frames omitted Suppressed: org.apache.ignite.IgniteCheckedException: Failed to connect to address [addr=grid596.ca.sbrf.ru/10.116.214.78:47100, err=Failed to read remote node recovery handshake (connection closed).] at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3463) ... 21 common frames omitted Caused by: org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException: Failed to read remote node recovery handshake (connection closed). at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.safeTcpHandshake(TcpCommunicationSpi.java:3753) at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.createTcpClient(TcpCommunicationSpi.java:3314) ... 21 common frames omitted