Ignite Command Line Startup, ver. 2.5.1-p12#20180807-sha1:8a4d3003 2018 Copyright(C) Apache Software Foundation [12:40:27,994][INFO][main][IgniteKernal] >>> __________ ________________ >>> / _/ ___/ |/ / _/_ __/ __/ >>> _/ // (7 7 // / / / / _/ >>> /___/\___/_/|_/___/ /_/ /___/ >>> >>> ver. ******#20180807-sha1:8a4d3003 >>> 2018 Copyright(C) Apache Software Foundation >>> >>> Ignite documentation: http://ignite.apache.org [12:40:27,995][INFO][main][IgniteKernal] Config URL: file:/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/server_default.xml [12:40:28,008][INFO][main][IgniteKernal] IgniteConfiguration [igniteInstanceName=null, pubPoolSize=16, svcPoolSize=16, callbackPoolSize=16, stripedPoolSize=16, sysPoolSize=16, mgmtPoolSize=4, igfsPoolSize=16, dataStreamerPoolSize=16, utilityCachePoolSize=16, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=16, igniteHome=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2, igniteWorkDir=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work, mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@6f94fa3e, nodeId=23245a3b-3b69-4129-ae50-2d6af3cd1232, marsh=org.apache.ignite.internal.binary.BinaryMarshaller@6bedbc4d, marshLocJobs=false, daemon=false, p2pEnabled=false, netTimeout=5000, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, sockTimeout=30000, ackTimeout=60000, marsh=null, reconCnt=2, reconDelay=2000, maxAckTimeout=300000, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null], segPlc=STOP, segResolveAttempts=2, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=10000, commSpi=TcpCommunicationSpi [connectGate=null, connPlc=null, enableForcibleNodeKill=true, enableTroubleshootingLog=false, srvLsnr=org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2@3514a4c0, locAddr=null, locHost=null, locPort=47100, locPortRange=100, shmemPort=-1, directBuf=true, directSndBuf=false, idleConnTimeout=600000, connTimeout=5000, maxConnTimeout=600000, reconCnt=10, sockSndBuf=32768, sockRcvBuf=32768, msgQueueLimit=0, slowClientQueueLimit=0, nioSrvr=null, shmemSrv=null, usePairedConnections=false, connectionsPerNode=1, tcpNoDelay=true, filterReachableAddresses=false, ackSndThreshold=32, unackedMsgsBufSize=0, sockWriteTimeout=2000, lsnr=null, boundTcpPort=-1, boundTcpShmemPort=-1, selectorsCnt=8, selectorSpins=0, addrRslvr=null, ctxInitLatch=java.util.concurrent.CountDownLatch@212b5695[Count = 1], stopping=false, metricsLsnr=org.apache.ignite.spi.communication.tcp.TcpCommunicationMetricsListener@446293d], evtSpi=org.apache.ignite.spi.eventstorage.NoopEventStorageSpi@69997e9d, colSpi=NoopCollisionSpi [], deploySpi=LocalDeploymentSpi [lsnr=null], indexingSpi=org.apache.ignite.spi.indexing.noop.NoopIndexingSpi@33990a0c, addrRslvr=null, clientMode=false, rebalanceThreadPoolSize=1, txCfg=org.apache.ignite.configuration.TransactionConfiguration@4de4b452, cacheSanityCheckEnabled=true, discoStartupDelay=60000, deployMode=SHARED, p2pMissedCacheSize=100, locHost=null, timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=10000, clientFailureDetectionTimeout=30000, metricsLogFreq=60000, hadoopCfg=null, connectorCfg=org.apache.ignite.configuration.ConnectorConfiguration@50b5ac82, odbcCfg=null, warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000, cacheMode=PARTITIONED, backups=1, aff=null, grpName=null], classLdr=null, sslCtxFactory=null, platformCfg=null, binaryCfg=BinaryConfiguration [idMapper=null, nameMapper=null, serializer=null, compactFooter=false], memCfg=null, pstCfg=null, dsCfg=DataStorageConfiguration [sysRegionInitSize=41943040, sysCacheMaxSize=104857600, pageSize=0, concLvl=0, dfltDataRegConf=DataRegionConfiguration [name=default, maxSize=20238390886, initSize=268435456, swapPath=null, pageEvictionMode=DISABLED, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=true, checkpointPageBufSize=0], storagePath=null, checkpointFreq=180000, lockWaitTime=10000, checkpointThreads=4, checkpointWriteOrder=SEQUENTIAL, walHistSize=20, walSegments=5, walSegmentSize=1000000, walPath=db/wal, walArchivePath=db/wal/archive, metricsEnabled=false, walMode=LOG_ONLY, walTlbSize=131072, walBuffSize=0, walFlushFreq=2000, walFsyncDelay=1000, walRecordIterBuffSize=67108864, alwaysWriteFullPages=false, fileIOFactory=org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory@6babf3bf, metricsSubIntervalCnt=5, metricsRateTimeInterval=60000, walAutoArchiveAfterInactivity=-1, writeThrottlingEnabled=false, walCompactionEnabled=false], activeOnStart=false, autoActivation=true, longQryWarnTimeout=3000, sqlConnCfg=null, cliConnCfg=ClientConnectorConfiguration [host=null, port=10800, portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true, maxOpenCursorsPerConn=128, threadPoolSize=16, idleTimeout=0, jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false, useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null], authEnabled=false, failureHnd=null, commFailureRslvr=null] [12:40:28,008][INFO][main][IgniteKernal] Daemon mode: off [12:40:28,009][INFO][main][IgniteKernal] OS: Linux 3.10.0-862.2.3.el7.x86_64 amd64 [12:40:28,009][INFO][main][IgniteKernal] OS user: dsherstobitov [12:40:28,009][INFO][main][IgniteKernal] PID: 25630 [12:40:28,009][INFO][main][IgniteKernal] Language runtime: Java Platform API Specification ver. 1.8 [12:40:28,009][INFO][main][IgniteKernal] VM information: Java(TM) SE Runtime Environment 1.8.0_161-b12 Oracle Corporation Java HotSpot(TM) 64-Bit Server VM 25.161-b12 [12:40:28,010][INFO][main][IgniteKernal] VM total memory: 31.0GB [12:40:28,010][INFO][main][IgniteKernal] Remote Management [restart: on, REST: on, JMX (remote: on, port: 50279, auth: off, ssl: off)] [12:40:28,011][INFO][main][IgniteKernal] Logger: JavaLogger [quiet=false, config=null] [12:40:28,011][INFO][main][IgniteKernal] IGNITE_HOME=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2 [12:40:28,011][INFO][main][IgniteKernal] VM arguments: [-Xms1g, -Xmx1g, -XX:+AggressiveOpts, -XX:MaxMetaspaceSize=256m, -DIGNITE_QUIET=false, -DIGNITE_SUCCESS_FILE=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/ignite_success_d1907159-d5bc-4f7f-9f36-d62d036c2cbe, -Dcom.sun.management.jmxremote, -Dcom.sun.management.jmxremote.port=50279, -Dcom.sun.management.jmxremote.authenticate=false, -Dcom.sun.management.jmxremote.ssl=false, -DIGNITE_HOME=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2, -DIGNITE_PROG_NAME=bin/ignite.sh, -Xmx31g, -Xms31g, -ea, -DIGNITE_DUMP_COUNTER_STACK=true, -DIGNITE_WAL_SERIALIZER_VERSION=2, -DIGNITE_WAL_LOG_TX_RECORDS=true, -DIGNITE_QUIET=false, -DIGNITE_NO_SHUTDOWN_HOOK=true, -DIGNITE_EXCHANGE_HISTORY_SIZE=18, -DIGNITE_ENABLE_FORCIBLE_NODE_KILL=true, -DIGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID=true, -DIGNITE_CLUSTER_NAME=ALMAZ, -DIGNITE_DIAGNOSTIC_ENABLED=false, -DIGNITE_DISABLE_WAL_DURING_REBALANCING=true, -DIGNITE_USE_ASYNC_FILE_IO_FACTORY=true, -XX:+UnlockExperimentalVMOptions, -XX:+UnlockDiagnosticVMOptions, -XX:+UnlockCommercialFeatures, -XX:OnOutOfMemoryError=kill;-3;%p, -XX:HeapDumpPath=/storage/ssd/dsherstobitov/tiden/, -XX:ErrorFile=/storage/ssd/dsherstobitov/tiden/hs_err%p.log, -XX:+UseG1GC, -DNODE_IP=172.25.1.45, -DNODE_COMMUNICATION_PORT=47101, -DCONSISTENT_ID=node2] [12:40:28,011][INFO][main][IgniteKernal] System cache's DataRegion size is configured to 40 MB. Use DataStorageConfiguration.systemCacheMemorySize property to change the setting. [12:40:28,012][INFO][main][IgniteKernal] Configured caches [in 'sysMemPlc' dataRegion: ['ignite-sys-cache'], in 'default' dataRegion: ['cache_group_1_001', 'cache_group_1_008', 'cache_group_1_015', 'cache_group_1_028', 'cache_group_2_031', 'cache_group_2_038', 'cache_group_2_045', 'cache_group_2_058', 'cache_group_3_061', 'cache_group_3_068', 'cache_group_3_075', 'cache_group_3_088', 'cache_group_4_091', 'cache_group_4_098', 'cache_group_4_105', 'cache_group_4_118']] [12:40:28,218][INFO][main][FailureProcessor] Configured failure handler: [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0]] [12:40:28,266][INFO][main][TcpCommunicationSpi] Successfully bound communication NIO server to TCP port [port=47101, locHost=0.0.0.0/0.0.0.0, selectorsCnt=8, selectorSpins=0, pairedConn=false] [12:40:28,328][WARNING][main][TcpCommunicationSpi] Message queue limit is set to 0 which may lead to potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and receiver sides. [12:40:28,346][WARNING][main][NoopCheckpointSpi] Checkpoints are disabled (to enable configure any GridCheckpointSpi implementation) [12:40:28,369][WARNING][main][GridCollisionManager] Collision resolution is disabled (all jobs will be activated upon arrival). [12:40:28,370][INFO][main][IgniteKernal] Security status [authentication=off, tls/ssl=off] [12:40:28,399][INFO][main][CacheObjectBinaryProcessorImpl] Resolved directory for serialized binary metadata: /storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/binary_meta/node2 [12:40:28,586][INFO][main][FilePageStoreManager] Resolved page store work directory: /storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/node2 [12:40:28,587][INFO][main][FileWriteAheadLogManager] Resolved write ahead log work directory: /storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2 [12:40:28,587][INFO][main][FileWriteAheadLogManager] Resolved write ahead log archive directory: /storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2 [12:40:28,634][INFO][main][FileWriteAheadLogManager] Started write-ahead log manager [mode=LOG_ONLY] [12:40:28,654][INFO][main][GridCacheDatabaseSharedManager] Read checkpoint status [startMarker=null, endMarker=null] [12:40:28,664][INFO][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=24800, tableSize=1.9 MiB, checkpointBuffer=100.0 MiB] [12:40:28,665][INFO][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=FileWALPointer [idx=0, fileOff=0, len=0], lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [12:40:28,676][INFO][main][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [12:40:28,676][INFO][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=0ms] [12:40:28,787][INFO][main][ClientListenerProcessor] Client connector processor has started on TCP port 10801 [12:40:28,833][INFO][main][GridTcpRestProtocol] Command protocol successfully started [name=TCP binary, host=0.0.0.0/0.0.0.0, port=11212] SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/ignite/libs/ignite-rest-http/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/ignite/libs/ignite-zookeeper/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] [12:40:29,013][INFO][main][GridJettyRestProtocol] Command protocol successfully started [name=Jetty REST, host=/0.0.0.0, port=8081] [12:40:29,052][INFO][main][IgniteKernal] Non-loopback local IPs: 172.17.0.1, 172.25.1.45, fe80:0:0:0:92e2:baff:fe07:b640%p2p1 [12:40:29,052][INFO][main][IgniteKernal] Enabled local MACs: 02424BB78199, 90E2BA07B640 [12:40:29,075][INFO][main][TcpDiscoverySpi] Successfully bound to TCP port [port=47501, localHost=0.0.0.0/0.0.0.0, locNodeId=23245a3b-3b69-4129-ae50-2d6af3cd1232] [12:40:29,136][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery accepted incoming connection [rmtAddr=/172.25.1.51, rmtPort=46825] [12:40:29,152][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery spawning a new thread for connection [rmtAddr=/172.25.1.51, rmtPort=46825] [12:40:29,152][INFO][tcp-disco-sock-reader-#4][TcpDiscoverySpi] Started serving remote node connection [rmtAddr=/172.25.1.51:46825, rmtPort=46825] [12:40:29,358][WARNING][main][IgniteKernal] Nodes started on local machine require more than 20% of physical RAM what can lead to significant slowdown due to swapping (please decrease JVM heap size, data region size or checkpoint buffer size) [required=106385MB, available=96504MB] [12:40:29,437][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=23245a3b-3b69-4129-ae50-2d6af3cd1232, customEvt=null, allowMerge=true] [12:40:29,440][INFO][exchange-worker-#62][FilePageStoreManager] Resolved page store work directory: /storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/node2 [12:40:29,441][INFO][exchange-worker-#62][FileWriteAheadLogManager] Resolved write ahead log work directory: /storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2 [12:40:29,441][INFO][exchange-worker-#62][FileWriteAheadLogManager] Resolved write ahead log archive directory: /storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2 [12:40:29,442][INFO][exchange-worker-#62][FileWriteAheadLogManager] Started write-ahead log manager [mode=LOG_ONLY] [12:40:29,453][INFO][exchange-worker-#62][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=24800, tableSize=1.9 MiB, checkpointBuffer=100.0 MiB] [12:40:29,549][INFO][exchange-worker-#62][PageMemoryImpl] Started page memory [memoryAllocated=18.8 GiB, pages=4789488, tableSize=372.6 MiB, checkpointBuffer=2.0 GiB] [12:40:29,552][INFO][exchange-worker-#62][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=24800, tableSize=1.9 MiB, checkpointBuffer=100.0 MiB] [12:40:29,658][INFO][exchange-worker-#62][GridCacheDatabaseSharedManager] Read checkpoint status [startMarker=null, endMarker=null] [12:40:29,659][INFO][exchange-worker-#62][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=FileWALPointer [idx=0, fileOff=0, len=0], lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [12:40:29,661][INFO][exchange-worker-#62][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, offset=0, ver=2] [12:40:29,688][INFO][exchange-worker-#62][GridSnapshotAwareClusterStateProcessorImpl] Writing BaselineTopology[id=0] [12:40:29,936][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=ignite-sys-cache, id=-2100569601, memoryPolicyName=sysMemPlc, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647] [12:40:29,964][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_1_001, id=1586135590, group=cache_group_1, memoryPolicyName=default, mode=PARTITIONED, atomicity=ATOMIC, backups=1] [12:40:29,971][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_1_008, id=1586135597, group=cache_group_2, memoryPolicyName=default, mode=PARTITIONED, atomicity=ATOMIC, backups=1] [12:40:29,976][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_1_015, id=1586135625, memoryPolicyName=default, mode=PARTITIONED, atomicity=ATOMIC, backups=1] [12:40:29,981][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_1_028, id=1586135659, memoryPolicyName=default, mode=PARTITIONED, atomicity=ATOMIC, backups=1] [12:40:29,985][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_2_031, id=1587059204, memoryPolicyName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647] [12:40:29,989][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_2_038, id=1587059211, group=cache_group_5, memoryPolicyName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647] [12:40:29,993][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_2_045, id=1587059239, group=cache_group_6, memoryPolicyName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647] [12:40:29,996][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_2_058, id=1587059273, memoryPolicyName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647] [12:40:29,998][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_3_061, id=1587982818, group=cache_group_1, memoryPolicyName=default, mode=PARTITIONED, atomicity=TRANSACTIONAL, backups=1] [12:40:30,000][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_3_068, id=1587982825, group=cache_group_2, memoryPolicyName=default, mode=PARTITIONED, atomicity=TRANSACTIONAL, backups=1] [12:40:30,003][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_3_075, id=1587982853, group=cache_group_3, memoryPolicyName=default, mode=PARTITIONED, atomicity=TRANSACTIONAL, backups=1] [12:40:30,006][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_3_088, id=1587982887, memoryPolicyName=default, mode=PARTITIONED, atomicity=TRANSACTIONAL, backups=1] [12:40:30,010][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_4_091, id=1588906432, group=cache_group_4, memoryPolicyName=default, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647] [12:40:30,012][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_4_098, id=1588906439, group=cache_group_5, memoryPolicyName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647] [12:40:30,014][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_4_105, id=1588907118, group=cache_group_6, memoryPolicyName=default, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647] [12:40:30,018][INFO][exchange-worker-#62][GridCacheProcessor] Started cache [name=cache_group_4_118, id=1588907152, memoryPolicyName=default, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647] [12:40:30,030][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finished waiting for partition release future [topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], waitTime=0ms, futInfo=NA] [12:40:30,040][INFO][grid-nio-worker-tcp-comm-0-#40][TcpCommunicationSpi] Established outgoing communication connection [locAddr=/0:0:0:0:0:0:0:1:59488, rmtAddr=/0:0:0:0:0:0:0:1%lo:47100] [12:40:30,042][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finished waiting for partition release future [topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], waitTime=0ms, futInfo=NA] [12:40:30,043][INFO][exchange-worker-#62][GridCacheDatabaseSharedManager] Read checkpoint status [startMarker=null, endMarker=null] [12:40:30,043][INFO][exchange-worker-#62][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [12:40:30,083][INFO][exchange-worker-#62][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=40ms] [12:40:30,099][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], crd=false] [12:40:30,099][INFO][db-checkpoint-thread-#66][GridCacheDatabaseSharedManager] Checkpoint started [checkpointId=ef470abc-8efd-4f98-86be-c0409fb68adb, startPtr=FileWALPointer [idx=0, fileOff=464991, len=119], checkpointLockWait=0ms, checkpointLockHoldTime=4ms, walCpRecordFsyncDuration=6ms, pages=119, reason='node started'] [12:40:30,147][INFO][db-checkpoint-thread-#66][GridCacheDatabaseSharedManager] Checkpoint finished [cpId=ef470abc-8efd-4f98-86be-c0409fb68adb, pages=119, markPos=FileWALPointer [idx=0, fileOff=464991, len=119], walSegmentsCleared=0, walSegmentsCovered=[], markDuration=15ms, pagesWrite=16ms, fsync=31ms, total=62ms] [12:40:30,188][INFO][sys-#71][GridDhtPartitionsExchangeFuture] Received full message, will finish exchange [node=784d8905-ba08-4a97-a3d5-99c3a33eccfb, resVer=AffinityTopologyVersion [topVer=14, minorTopVer=0]] [12:40:30,345][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=0, segIdx=0, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000000.wal] [12:40:30,358][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000000.wal] [12:40:30,438][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=1, segIdx=1, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000001.wal] [12:40:30,450][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000001.wal] [12:40:30,517][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=2, segIdx=2, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000002.wal] [12:40:30,529][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000002.wal] [12:40:30,585][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=3, segIdx=3, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000003.wal] [12:40:30,597][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000003.wal] [12:40:30,651][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=4, segIdx=4, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000004.wal] [12:40:30,663][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000004.wal] [12:40:30,702][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=5, segIdx=0, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000005.wal] [12:40:30,716][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000005.wal] [12:40:30,757][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=6, segIdx=1, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000006.wal] [12:40:30,771][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000006.wal] [12:40:30,807][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=7, segIdx=2, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000007.wal] [12:40:30,822][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000007.wal] [12:40:30,873][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=8, segIdx=3, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000008.wal] [12:40:30,876][INFO][sys-#71][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], err=null] [12:40:30,887][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000008.wal] [12:40:30,892][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_1_028 changed from true to false [12:40:30,894][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_3_088 changed from true to false [12:40:30,894][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_1_015 changed from true to false [12:40:30,895][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_4_118 changed from true to false [12:40:30,896][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_2_058 changed from true to false [12:40:30,896][INFO][sys-#71][CacheGroupContext] Local WAL state for group=ignite-sys-cache changed from true to false [12:40:30,897][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_6 changed from true to false [12:40:30,897][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_5 changed from true to false [12:40:30,898][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_4 changed from true to false [12:40:30,899][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_3 changed from true to false [12:40:30,899][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_2 changed from true to false [12:40:30,900][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_1 changed from true to false [12:40:30,900][INFO][sys-#71][CacheGroupContext] Local WAL state for group=cache_group_2_031 changed from true to false [12:40:30,952][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Rebalancing scheduled [order=[ignite-sys-cache, cache_group_2_031, cache_group_1, cache_group_2, cache_group_3, cache_group_4, cache_group_5, cache_group_6, cache_group_2_058, cache_group_4_118, cache_group_1_015, cache_group_3_088, cache_group_1_028], top=AffinityTopologyVersion [topVer=14, minorTopVer=0], force=false, evt=NODE_JOINED, node=23245a3b-3b69-4129-ae50-2d6af3cd1232] [12:40:30,954][INFO][exchange-worker-#62][GridDhtPartitionDemander] Starting rebalancing [grp=ignite-sys-cache, mode=SYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=34, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:30,960][INFO][exchange-worker-#62][GridDhtPartitionDemander] Starting rebalancing [grp=ignite-sys-cache, mode=SYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=35, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:30,960][INFO][exchange-worker-#62][GridDhtPartitionDemander] Starting rebalancing [grp=ignite-sys-cache, mode=SYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=31, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:30,963][INFO][grid-nio-worker-tcp-comm-1-#41][TcpCommunicationSpi] Established outgoing communication connection [locAddr=/172.25.1.45:53188, rmtAddr=/172.25.1.51:47101] [12:40:30,973][INFO][utility-#76][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=ignite-sys-cache, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=20 ms] [12:40:30,974][INFO][utility-#75][GridDhtPartitionDemander] Completed rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=ignite-sys-cache, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=20 ms] [12:40:31,767][INFO][grid-nio-worker-tcp-comm-2-#42][TcpCommunicationSpi] Established outgoing communication connection [locAddr=/172.25.1.45:58298, rmtAddr=/172.25.1.51:47100] [12:40:31,776][INFO][utility-#77][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=ignite-sys-cache, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=824 ms] [12:40:31,777][INFO][utility-#77][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=ignite-sys-cache], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:31,778][INFO][utility-#77][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2_031, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=174, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:31,779][INFO][utility-#77][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2_031, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=172, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:31,780][INFO][utility-#77][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2_031, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=166, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:31,835][INFO][main][IgniteKernal] Performance suggestions for grid (fix if possible) [12:40:31,835][INFO][main][IgniteKernal] To disable, set -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true [12:40:31,835][INFO][main][IgniteKernal] ^-- Set max direct memory size if getting 'OOME: Direct buffer memory' (add '-XX:MaxDirectMemorySize=[g|G|m|M|k|K]' to JVM options) [12:40:31,835][INFO][main][IgniteKernal] ^-- Disable processing of calls to System.gc() (add '-XX:+DisableExplicitGC' to JVM options) [12:40:31,835][INFO][main][IgniteKernal] ^-- Speed up flushing of dirty pages by OS (alter vm.dirty_expire_centisecs parameter by setting to 500) [12:40:31,835][INFO][main][IgniteKernal] ^-- Decrease number of backups (set 'backups' to 0) [12:40:31,835][INFO][main][IgniteKernal] ^-- Disable fully synchronous writes (set 'writeSynchronizationMode' to PRIMARY_SYNC or FULL_ASYNC) [12:40:31,835][INFO][main][IgniteKernal] ^-- Disable assertions (remove '-ea' from JVM options) [12:40:31,835][INFO][main][IgniteKernal] Refer to this page for more performance suggestions: https://apacheignite.readme.io/docs/jvm-and-system-tuning [12:40:31,835][INFO][main][IgniteKernal] [12:40:31,835][INFO][main][IgniteKernal] To start Console Management & Monitoring run ignitevisorcmd.{sh|bat} [12:40:31,835][INFO][main][IgniteKernal] [12:40:31,836][INFO][main][IgniteKernal] >>> +--------------------------------------------------------------------------+ >>> Ignite ver. 2.5.1-p12#20180807-sha1:8a4d300316774389b2a0ce5d8f41b3b53ebc8e49 >>> +--------------------------------------------------------------------------+ >>> OS name: Linux 3.10.0-862.2.3.el7.x86_64 amd64 >>> CPU(s): 16 >>> Heap: 31.0GB >>> VM name: 25630@lab45.*.local >>> Local node [ID=23245A3B-3B69-4129-AE50-2D6AF3CD1232, order=14, clientMode=false] >>> Local node addresses: [172.17.0.1/0:0:0:0:0:0:0:1%lo, /172.17.0.1, /172.25.1.45] >>> Local ports: TCP:8081 TCP:10801 TCP:11212 TCP:47101 TCP:47501 [12:40:31,836][INFO][main][GridDiscoveryManager] Topology snapshot [ver=14, servers=4, clients=0, CPUs=32, offheap=75.0GB, heap=120.0GB] [12:40:31,836][INFO][main][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:31,837][INFO][main][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:31,837][INFO][main][GridDiscoveryManager] Data Regions Configured: [12:40:31,837][INFO][main][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:32,692][INFO][sys-#85][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_2_031, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=916 ms] [12:40:32,701][INFO][sys-#84][GridDhtPartitionDemander] Completed rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_2_031, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=926 ms] [12:40:32,721][INFO][sys-#83][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_2_031, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=946 ms] [12:40:32,721][INFO][sys-#83][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_2_031], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:32,721][INFO][sys-#83][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=176, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:32,722][INFO][sys-#83][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=173, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:32,723][INFO][sys-#83][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=132, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,152][INFO][sys-#87][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_1, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=433 ms] [12:40:33,205][INFO][sys-#86][GridDhtPartitionDemander] Completed rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_1, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=483 ms] [12:40:33,217][INFO][sys-#88][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_1, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=493 ms] [12:40:33,218][INFO][sys-#88][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_1], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,218][INFO][sys-#88][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=25, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,218][INFO][sys-#88][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=27, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,218][INFO][sys-#88][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=21, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,632][INFO][sys-#83][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_2, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=414 ms] [12:40:33,692][INFO][sys-#79][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_2, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=474 ms] [12:40:33,738][INFO][sys-#82][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_2, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=524 ms] [12:40:33,738][INFO][sys-#82][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_2], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,739][INFO][sys-#82][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_3, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=11, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,739][INFO][sys-#82][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_3, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=16, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,740][INFO][sys-#82][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_3, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=10, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:33,756][WARNING][sys-#74][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_3_075 [12:40:34,028][INFO][sys-#74][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_3, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=284 ms] [12:40:34,035][INFO][sys-#72][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_3, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=294 ms] [12:40:34,128][INFO][sys-#85][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_3, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=384 ms] [12:40:34,128][INFO][sys-#85][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_3], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:34,129][INFO][sys-#85][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_4, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=148, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:34,129][INFO][sys-#85][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_4, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=175, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:34,130][INFO][sys-#85][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_4, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=189, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:34,519][INFO][sys-#71][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_4, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=393 ms] [12:40:34,535][INFO][sys-#82][GridDhtPartitionDemander] Completed rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_4, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=413 ms] [12:40:34,573][INFO][sys-#78][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_4, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=443 ms] [12:40:34,573][INFO][sys-#78][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_4], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:34,573][INFO][sys-#78][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_5, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=41, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:34,574][INFO][sys-#78][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_5, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=42, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:34,574][INFO][sys-#78][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_5, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=45, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:35,143][INFO][sys-#86][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_5, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=564 ms] [12:40:35,211][INFO][sys-#88][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_5, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=634 ms] [12:40:35,231][INFO][sys-#73][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_5, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=654 ms] [12:40:35,231][INFO][sys-#73][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_5], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:35,232][INFO][sys-#73][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_6, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=21, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:35,232][INFO][sys-#73][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_6, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=25, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:35,233][INFO][sys-#73][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_6, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=18, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:35,245][WARNING][sys-#83][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_2_045 [12:40:35,266][WARNING][sys-#83][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_4_105 [12:40:36,077][INFO][sys-#87][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_6, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=842 ms] [12:40:36,144][INFO][sys-#78][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_6, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=912 ms] [12:40:36,282][INFO][sys-#83][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_6, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=1053 ms] [12:40:36,282][INFO][sys-#83][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_6], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,283][INFO][sys-#83][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2_058, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=5, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,283][INFO][sys-#83][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2_058, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=12, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,283][INFO][sys-#83][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_2_058, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=15, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,293][WARNING][sys-#65][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_2_058 [12:40:36,476][INFO][sys-#87][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_2_058, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=191 ms] [12:40:36,735][INFO][sys-#65][GridDhtPartitionDemander] Completed rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_2_058, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=453 ms] [12:40:36,797][INFO][sys-#74][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_2_058, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=513 ms] [12:40:36,797][INFO][sys-#74][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_2_058], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,797][INFO][sys-#74][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_4_118, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=16, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,797][INFO][sys-#74][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_4_118, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=10, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,798][INFO][sys-#74][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_4_118, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=6, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:36,806][WARNING][sys-#86][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_4_118 [12:40:37,014][INFO][sys-#83][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_4_118, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=211 ms] [12:40:37,148][INFO][sys-#84][GridDhtPartitionDemander] Completed rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_4_118, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=352 ms] [12:40:37,326][INFO][sys-#86][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_4_118, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=523 ms] [12:40:37,326][INFO][sys-#86][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_4_118], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,326][INFO][sys-#86][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1_015, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=11, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,326][INFO][sys-#86][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1_015, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=16, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,326][INFO][sys-#86][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1_015, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=10, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,335][WARNING][sys-#74][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_1_015 [12:40:37,506][INFO][sys-#74][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_1_015, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=181 ms] [12:40:37,518][INFO][sys-#72][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_1_015, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=201 ms] [12:40:37,580][INFO][sys-#73][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_1_015, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=262 ms] [12:40:37,580][INFO][sys-#73][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_1_015], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,580][INFO][sys-#73][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_3_088, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=4, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,580][INFO][sys-#73][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_3_088, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=10, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,581][INFO][sys-#73][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_3_088, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=5, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,589][WARNING][sys-#86][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_3_088 [12:40:37,730][INFO][sys-#86][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_3_088, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=150 ms] [12:40:37,770][INFO][sys-#85][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_3_088, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=191 ms] [12:40:37,970][INFO][sys-#72][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_3_088, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=387 ms] [12:40:37,970][INFO][sys-#72][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_3_088], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,970][INFO][sys-#72][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1_028, mode=ASYNC, fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, partitionsCount=4, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,971][INFO][sys-#72][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1_028, mode=ASYNC, fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, partitionsCount=10, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,971][INFO][sys-#72][GridDhtPartitionDemander] Starting rebalancing [grp=cache_group_1_028, mode=ASYNC, fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, partitionsCount=5, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:37,980][WARNING][sys-#80][GridCacheEvictionManager] Evictions started (cache may have reached its capacity). You may wish to increase 'maxSize' on eviction policy being used for cache: cache_group_1_028 [12:40:38,096][INFO][sys-#82][GridDhtPartitionDemander] Completed rebalancing [fromNode=ce360471-0a95-466b-baac-65cd6ddddce4, cacheOrGroup=cache_group_1_028, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=131 ms] [12:40:38,131][INFO][sys-#73][GridDhtPartitionDemander] Completed rebalancing [fromNode=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, cacheOrGroup=cache_group_1_028, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=161 ms] [12:40:38,268][INFO][sys-#80][GridDhtPartitionDemander] Completed (final) rebalancing [fromNode=784d8905-ba08-4a97-a3d5-99c3a33eccfb, cacheOrGroup=cache_group_1_028, topology=AffinityTopologyVersion [topVer=14, minorTopVer=0], time=302 ms] [12:40:38,268][INFO][sys-#80][GridDhtPartitionDemander] Completed rebalance future: RebalanceFuture [grp=CacheGroupContext [grp=cache_group_1_028], topVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], rebalanceId=1] [12:40:38,268][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_1_028 changed from false to true [12:40:38,269][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_3_088 changed from false to true [12:40:38,269][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_1_015 changed from false to true [12:40:38,269][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_4_118 changed from false to true [12:40:38,270][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_2_058 changed from false to true [12:40:38,270][INFO][sys-#80][CacheGroupContext] Local WAL state for group=ignite-sys-cache changed from false to true [12:40:38,270][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_6 changed from false to true [12:40:38,270][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_5 changed from false to true [12:40:38,270][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_4 changed from false to true [12:40:38,271][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_3 changed from false to true [12:40:38,271][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_2 changed from false to true [12:40:38,271][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_1 changed from false to true [12:40:38,271][INFO][sys-#80][CacheGroupContext] Local WAL state for group=cache_group_2_031 changed from false to true [12:40:38,294][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=9, segIdx=4, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000009.wal] [12:40:38,314][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000009.wal] [12:40:38,315][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=10, segIdx=0, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000010.wal] [12:40:38,335][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000010.wal] [12:40:38,335][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=11, segIdx=1, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000011.wal] [12:40:38,355][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000011.wal] [12:40:38,355][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=12, segIdx=2, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000012.wal] [12:40:38,375][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000012.wal] [12:40:38,375][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=13, segIdx=3, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000013.wal] [12:40:38,395][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000013.wal] [12:40:38,395][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=14, segIdx=4, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000014.wal] [12:40:38,415][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000014.wal] [12:40:38,416][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=15, segIdx=0, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000015.wal] [12:40:38,436][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000015.wal] [12:40:38,436][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=16, segIdx=1, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000016.wal] [12:40:38,456][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000016.wal] [12:40:38,456][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=17, segIdx=2, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000017.wal] [12:40:38,476][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000002.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000017.wal] [12:40:38,476][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=18, segIdx=3, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000018.wal] [12:40:38,496][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000003.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000018.wal] [12:40:38,496][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=19, segIdx=4, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000019.wal] [12:40:38,516][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000004.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000019.wal] [12:40:38,516][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=20, segIdx=0, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000020.wal] [12:40:38,535][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000000.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000020.wal] [12:40:38,536][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Starting to copy WAL segment [absIdx=21, segIdx=1, origFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dstFile=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000021.wal] [12:40:38,552][INFO][wal-file-archiver%null-#63][FileWriteAheadLogManager] Copied file [src=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/node2/0000000000000001.wal, dst=/storage/ssd/dsherstobitov/tiden/rebalance-180814-123832/test_rebalance/ignite.server.2/work/db/wal/archive/node2/0000000000000021.wal] [12:40:38,744][INFO][db-checkpoint-thread-#66][GridCacheDatabaseSharedManager] Checkpoint started [checkpointId=48f255dc-5070-4117-a4b1-90f10f0f8705, startPtr=FileWALPointer [idx=22, fileOff=608737, len=38993], checkpointLockWait=0ms, checkpointLockHoldTime=270ms, walCpRecordFsyncDuration=10ms, pages=154340, reason='wal-local-state-changed-rebalance-finished-AffinityTopologyVersion [topVer=14, minorTopVer=0]'] [12:40:43,547][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=15, intOrder=10, lastExchangeTime=1534239643464, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:43,547][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=15, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:40:43,547][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:43,547][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:43,548][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:43,548][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:43,549][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, customEvt=null, allowMerge=true] [12:40:43,550][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], err=null] [12:40:43,551][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_1_028 changed from true to false [12:40:43,552][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_3_088 changed from true to false [12:40:43,552][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_1_015 changed from true to false [12:40:43,553][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_4_118 changed from true to false [12:40:43,553][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_2_058 changed from true to false [12:40:43,553][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=ignite-sys-cache changed from true to false [12:40:43,554][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_6 changed from true to false [12:40:43,554][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_5 changed from true to false [12:40:43,554][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_4 changed from true to false [12:40:43,555][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_3 changed from true to false [12:40:43,555][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_2 changed from true to false [12:40:43,555][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_1 changed from true to false [12:40:43,555][INFO][exchange-worker-#62][CacheGroupContext] Local WAL state for group=cache_group_2_031 changed from true to false [12:40:43,556][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=15, minorTopVer=0], crd=false] [12:40:43,562][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=15, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, client=false] [12:40:43,577][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=92d86d56-346f-4aac-9d97-a61b22c2ce08, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=16, intOrder=11, lastExchangeTime=1534239643555, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:43,578][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=16, servers=4, clients=2, CPUs=48, offheap=110.0GB, heap=170.0GB] [12:40:43,578][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:43,578][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:43,578][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:43,578][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:43,578][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=16, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=92d86d56-346f-4aac-9d97-a61b22c2ce08, customEvt=null, allowMerge=true] [12:40:43,579][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=16, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=16, minorTopVer=0], err=null] [12:40:43,580][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=16, minorTopVer=0], crd=false] [12:40:43,585][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=16, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=92d86d56-346f-4aac-9d97-a61b22c2ce08, client=false] [12:40:44,465][INFO][grid-nio-worker-tcp-comm-3-#43][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45364] [12:40:44,667][INFO][grid-nio-worker-tcp-comm-4-#44][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45366] [12:40:44,868][INFO][grid-nio-worker-tcp-comm-5-#45][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45368] [12:40:45,069][INFO][grid-nio-worker-tcp-comm-6-#46][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45370] [12:40:45,270][INFO][grid-nio-worker-tcp-comm-7-#47][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45372] [12:40:45,271][INFO][grid-nio-worker-tcp-comm-0-#40][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/0:0:0:0:0:0:0:1:47101, rmtAddr=/0:0:0:0:0:0:0:1:54716] [12:40:45,271][INFO][grid-nio-worker-tcp-comm-1-#41][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/127.0.0.1:47101, rmtAddr=/127.0.0.1:48860] [12:40:45,277][WARNING][tcp-disco-msg-worker-#2][TcpDiscoverySpi] Received EVT_NODE_FAILED event with warning [nodeInitiatedEvt=TcpDiscoveryNode [id=784d8905-ba08-4a97-a3d5-99c3a33eccfb, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.45], sockAddrs=[lab45.*.local/172.25.1.45:47500, /172.17.0.1:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1534239629226, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false], msg=TcpCommunicationSpi failed to establish connection to node [rmtNode=TcpDiscoveryNode [id=92d86d56-346f-4aac-9d97-a61b22c2ce08, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=16, intOrder=11, lastExchangeTime=1534239643535, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true], errs=class o.a.i.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=92d86d56-346f-4aac-9d97-a61b22c2ce08, addrs=[/172.25.1.50:47101, /172.17.0.1:47101, /0:0:0:0:0:0:0:1%lo:47101, /127.0.0.1:47101]], connectErrs=[class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.25.1.50:47101, err=Connection refused], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=92d86d56-346f-4aac-9d97-a61b22c2ce08, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=92d86d56-346f-4aac-9d97-a61b22c2ce08, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=92d86d56-346f-4aac-9d97-a61b22c2ce08, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=92d86d56-346f-4aac-9d97-a61b22c2ce08, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=92d86d56-346f-4aac-9d97-a61b22c2ce08, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/0:0:0:0:0:0:0:1%lo:47101, err=Remote node ID is not as expected [expected=92d86d56-346f-4aac-9d97-a61b22c2ce08, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/127.0.0.1:47101, err=Remote node ID is not as expected [expected=92d86d56-346f-4aac-9d97-a61b22c2ce08, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]]]]] [12:40:45,279][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=92d86d56-346f-4aac-9d97-a61b22c2ce08, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=16, intOrder=11, lastExchangeTime=1534239643555, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:45,279][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=17, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:40:45,279][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:45,279][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:45,279][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:45,279][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:45,285][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=17, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=92d86d56-346f-4aac-9d97-a61b22c2ce08, customEvt=null, allowMerge=true] [12:40:45,285][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=17, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=17, minorTopVer=0], err=null] [12:40:45,286][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=17, minorTopVer=0], crd=false] [12:40:45,291][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=17, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=92d86d56-346f-4aac-9d97-a61b22c2ce08, client=false] [12:40:46,081][WARNING][tcp-disco-msg-worker-#2][TcpDiscoverySpi] Received EVT_NODE_FAILED event with warning [nodeInitiatedEvt=TcpDiscoveryNode [id=784d8905-ba08-4a97-a3d5-99c3a33eccfb, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.45], sockAddrs=[lab45.*.local/172.25.1.45:47500, /172.17.0.1:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1534239629226, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false], msg=TcpCommunicationSpi failed to establish connection to node [rmtNode=TcpDiscoveryNode [id=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=15, intOrder=10, lastExchangeTime=1534239643273, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true], errs=class o.a.i.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, addrs=[/172.17.0.1:47100, /172.25.1.50:47100, /0:0:0:0:0:0:0:1%lo:47100, /127.0.0.1:47100]], connectErrs=[class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.25.1.50:47100, err=Connection refused]]]] [12:40:46,082][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=15, intOrder=10, lastExchangeTime=1534239643464, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:46,082][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=18, servers=4, clients=0, CPUs=32, offheap=75.0GB, heap=120.0GB] [12:40:46,082][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:46,082][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:46,082][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:46,082][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:46,083][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=18, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, customEvt=null, allowMerge=true] [12:40:46,084][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=18, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=18, minorTopVer=0], err=null] [12:40:46,085][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=18, minorTopVer=0], crd=false] [12:40:46,089][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=18, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=9d137119-8d8a-4b9a-8e1f-1b6cbda1a5e6, client=false] [12:40:48,791][INFO][db-checkpoint-thread-#66][GridCacheDatabaseSharedManager] Checkpoint finished [cpId=48f255dc-5070-4117-a4b1-90f10f0f8705, pages=154340, markPos=FileWALPointer [idx=22, fileOff=608737, len=38993], walSegmentsCleared=0, walSegmentsCovered=[0 - 21], markDuration=472ms, pagesWrite=1245ms, fsync=8802ms, total=10519ms] [12:40:52,844][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery accepted incoming connection [rmtAddr=/172.25.1.50, rmtPort=45478] [12:40:52,844][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery spawning a new thread for connection [rmtAddr=/172.25.1.50, rmtPort=45478] [12:40:52,844][INFO][tcp-disco-sock-reader-#5][TcpDiscoverySpi] Started serving remote node connection [rmtAddr=/172.25.1.50:45478, rmtPort=45478] [12:40:52,909][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=486f1997-0f4e-4dad-861c-896dfaef4348, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=19, intOrder=12, lastExchangeTime=1534239652886, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:52,909][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=19, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:40:52,909][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:52,909][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:52,909][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:52,910][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:52,910][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=19, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=486f1997-0f4e-4dad-861c-896dfaef4348, customEvt=null, allowMerge=true] [12:40:52,911][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=19, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=19, minorTopVer=0], err=null] [12:40:52,911][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=19, minorTopVer=0], crd=false] [12:40:52,916][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=19, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=486f1997-0f4e-4dad-861c-896dfaef4348, client=false] [12:40:52,942][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=80b06496-f6a6-43d3-bddc-e1356046d19e, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=20, intOrder=13, lastExchangeTime=1534239652926, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:52,943][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=20, servers=4, clients=2, CPUs=48, offheap=110.0GB, heap=170.0GB] [12:40:52,943][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:52,943][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:52,943][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:52,943][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:52,943][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=80b06496-f6a6-43d3-bddc-e1356046d19e, customEvt=null, allowMerge=true] [12:40:52,944][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], err=null] [12:40:52,944][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], crd=false] [12:40:52,948][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=20, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=80b06496-f6a6-43d3-bddc-e1356046d19e, client=false] [12:40:53,820][INFO][grid-nio-worker-tcp-comm-2-#42][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45392] [12:40:53,876][INFO][tcp-disco-sock-reader-#5][TcpDiscoverySpi] Finished serving remote node connection [rmtAddr=/172.25.1.50:45478, rmtPort=45478 [12:40:54,020][INFO][grid-nio-worker-tcp-comm-3-#43][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45394] [12:40:54,221][INFO][grid-nio-worker-tcp-comm-4-#44][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45396] [12:40:54,421][INFO][grid-nio-worker-tcp-comm-5-#45][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45398] [12:40:54,622][INFO][grid-nio-worker-tcp-comm-6-#46][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45400] [12:40:54,623][INFO][grid-nio-worker-tcp-comm-7-#47][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/0:0:0:0:0:0:0:1:47101, rmtAddr=/0:0:0:0:0:0:0:1:54744] [12:40:54,623][INFO][grid-nio-worker-tcp-comm-0-#40][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/127.0.0.1:47101, rmtAddr=/127.0.0.1:48888] [12:40:54,628][WARNING][tcp-disco-msg-worker-#2][TcpDiscoverySpi] Received EVT_NODE_FAILED event with warning [nodeInitiatedEvt=TcpDiscoveryNode [id=784d8905-ba08-4a97-a3d5-99c3a33eccfb, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.45], sockAddrs=[lab45.*.local/172.25.1.45:47500, /172.17.0.1:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1534239629226, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false], msg=TcpCommunicationSpi failed to establish connection to node [rmtNode=TcpDiscoveryNode [id=80b06496-f6a6-43d3-bddc-e1356046d19e, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=20, intOrder=13, lastExchangeTime=1534239652906, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true], errs=class o.a.i.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=80b06496-f6a6-43d3-bddc-e1356046d19e, addrs=[/172.25.1.50:47101, /172.17.0.1:47101, /0:0:0:0:0:0:0:1%lo:47101, /127.0.0.1:47101]], connectErrs=[class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.25.1.50:47101, err=Connection refused], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=80b06496-f6a6-43d3-bddc-e1356046d19e, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=80b06496-f6a6-43d3-bddc-e1356046d19e, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=80b06496-f6a6-43d3-bddc-e1356046d19e, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=80b06496-f6a6-43d3-bddc-e1356046d19e, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=80b06496-f6a6-43d3-bddc-e1356046d19e, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/0:0:0:0:0:0:0:1%lo:47101, err=Remote node ID is not as expected [expected=80b06496-f6a6-43d3-bddc-e1356046d19e, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/127.0.0.1:47101, err=Remote node ID is not as expected [expected=80b06496-f6a6-43d3-bddc-e1356046d19e, rcvd=23245a3b-3b69-4129-ae50-2d6af3cd1232]]]]] [12:40:54,629][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=80b06496-f6a6-43d3-bddc-e1356046d19e, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=20, intOrder=13, lastExchangeTime=1534239652926, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:54,629][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=21, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:40:54,629][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:54,629][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:54,629][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:54,629][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:54,630][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=21, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=80b06496-f6a6-43d3-bddc-e1356046d19e, customEvt=null, allowMerge=true] [12:40:54,630][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=21, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=21, minorTopVer=0], err=null] [12:40:54,631][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=21, minorTopVer=0], crd=false] [12:40:54,635][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=21, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=80b06496-f6a6-43d3-bddc-e1356046d19e, client=false] [12:40:55,430][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery accepted incoming connection [rmtAddr=/0:0:0:0:0:0:0:1, rmtPort=54127] [12:40:55,430][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery spawning a new thread for connection [rmtAddr=/0:0:0:0:0:0:0:1, rmtPort=54127] [12:40:55,430][INFO][tcp-disco-sock-reader-#7][TcpDiscoverySpi] Started serving remote node connection [rmtAddr=/0:0:0:0:0:0:0:1:54127, rmtPort=54127] [12:40:55,431][INFO][tcp-disco-sock-reader-#7][TcpDiscoverySpi] Received ping request from the remote node [rmtNodeId=784d8905-ba08-4a97-a3d5-99c3a33eccfb, rmtAddr=/0:0:0:0:0:0:0:1:54127, rmtPort=54127] [12:40:55,432][INFO][tcp-disco-sock-reader-#7][TcpDiscoverySpi] Finished writing ping response [rmtNodeId=784d8905-ba08-4a97-a3d5-99c3a33eccfb, rmtAddr=/0:0:0:0:0:0:0:1:54127, rmtPort=54127] [12:40:55,432][INFO][tcp-disco-sock-reader-#7][TcpDiscoverySpi] Finished serving remote node connection [rmtAddr=/0:0:0:0:0:0:0:1:54127, rmtPort=54127 [12:40:55,433][WARNING][tcp-disco-msg-worker-#2][TcpDiscoverySpi] Received EVT_NODE_FAILED event with warning [nodeInitiatedEvt=TcpDiscoveryNode [id=784d8905-ba08-4a97-a3d5-99c3a33eccfb, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.45], sockAddrs=[lab45.*.local/172.25.1.45:47500, /172.17.0.1:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1534239629226, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false], msg=TcpCommunicationSpi failed to establish connection to node [rmtNode=TcpDiscoveryNode [id=486f1997-0f4e-4dad-861c-896dfaef4348, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=19, intOrder=12, lastExchangeTime=1534239652876, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true], errs=class o.a.i.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=486f1997-0f4e-4dad-861c-896dfaef4348, addrs=[/172.17.0.1:47100, /172.25.1.50:47100, /127.0.0.1:47100, /0:0:0:0:0:0:0:1%lo:47100]], connectErrs=[class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=486f1997-0f4e-4dad-861c-896dfaef4348, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=486f1997-0f4e-4dad-861c-896dfaef4348, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=486f1997-0f4e-4dad-861c-896dfaef4348, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=486f1997-0f4e-4dad-861c-896dfaef4348, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=486f1997-0f4e-4dad-861c-896dfaef4348, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.25.1.50:47100, err=Connection refused]]]] [12:40:55,433][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=486f1997-0f4e-4dad-861c-896dfaef4348, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=19, intOrder=12, lastExchangeTime=1534239652886, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:40:55,434][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=22, servers=4, clients=0, CPUs=32, offheap=75.0GB, heap=120.0GB] [12:40:55,434][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:40:55,434][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:40:55,434][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:40:55,434][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:40:55,435][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=22, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=486f1997-0f4e-4dad-861c-896dfaef4348, customEvt=null, allowMerge=true] [12:40:55,435][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=22, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=22, minorTopVer=0], err=null] [12:40:55,436][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=22, minorTopVer=0], crd=false] [12:40:55,439][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=22, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=486f1997-0f4e-4dad-861c-896dfaef4348, client=false] [12:41:01,541][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery accepted incoming connection [rmtAddr=/172.25.1.50, rmtPort=39871] [12:41:01,541][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery spawning a new thread for connection [rmtAddr=/172.25.1.50, rmtPort=39871] [12:41:01,541][INFO][tcp-disco-sock-reader-#8][TcpDiscoverySpi] Started serving remote node connection [rmtAddr=/172.25.1.50:39871, rmtPort=39871] [12:41:01,694][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=56e1b7b7-f18f-4875-9812-a3c7e21c2742, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=23, intOrder=14, lastExchangeTime=1534239661663, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:01,694][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=23, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:41:01,694][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:01,695][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:01,695][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:01,695][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:01,695][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=23, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=56e1b7b7-f18f-4875-9812-a3c7e21c2742, customEvt=null, allowMerge=true] [12:41:01,696][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=23, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=23, minorTopVer=0], err=null] [12:41:01,696][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=23, minorTopVer=0], crd=false] [12:41:01,702][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=23, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=56e1b7b7-f18f-4875-9812-a3c7e21c2742, client=false] [12:41:01,704][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=24, intOrder=15, lastExchangeTime=1534239661684, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:01,704][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=24, servers=4, clients=2, CPUs=48, offheap=110.0GB, heap=170.0GB] [12:41:01,704][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:01,704][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:01,704][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:01,704][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:01,705][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=24, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, customEvt=null, allowMerge=true] [12:41:01,705][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=24, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=24, minorTopVer=0], err=null] [12:41:01,706][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=24, minorTopVer=0], crd=false] [12:41:01,709][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=24, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, client=false] [12:41:02,436][INFO][grid-nio-worker-tcp-comm-1-#41][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.25.1.45:47101, rmtAddr=/172.25.1.50:36100] [12:41:02,718][INFO][tcp-disco-sock-reader-#8][TcpDiscoverySpi] Finished serving remote node connection [rmtAddr=/172.25.1.50:39871, rmtPort=39871 [12:41:02,722][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] Connect timed out (consider increasing 'failureDetectionTimeout' configuration property) [addr=/172.25.1.50:47101, failureDetectionTimeout=10000] [12:41:02,723][INFO][grid-nio-worker-tcp-comm-2-#42][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45422] [12:41:02,923][INFO][grid-nio-worker-tcp-comm-3-#43][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45426] [12:41:03,124][INFO][grid-nio-worker-tcp-comm-4-#44][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45430] [12:41:03,325][INFO][grid-nio-worker-tcp-comm-5-#45][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45434] [12:41:03,371][INFO][grid-nio-worker-tcp-comm-6-#46][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45442] [12:41:03,374][WARNING][tcp-disco-msg-worker-#2][TcpDiscoverySpi] Received EVT_NODE_FAILED event with warning [nodeInitiatedEvt=TcpDiscoveryNode [id=784d8905-ba08-4a97-a3d5-99c3a33eccfb, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.45], sockAddrs=[lab45.*.local/172.25.1.45:47500, /172.17.0.1:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1534239629226, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false], msg=TcpCommunicationSpi failed to establish connection to node [rmtNode=TcpDiscoveryNode [id=56e1b7b7-f18f-4875-9812-a3c7e21c2742, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=23, intOrder=14, lastExchangeTime=1534239661593, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true], errs=class o.a.i.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=56e1b7b7-f18f-4875-9812-a3c7e21c2742, addrs=[/172.17.0.1:47100, /172.25.1.50:47100, /127.0.0.1:47100, /0:0:0:0:0:0:0:1%lo:47100]], connectErrs=[class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=56e1b7b7-f18f-4875-9812-a3c7e21c2742, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=56e1b7b7-f18f-4875-9812-a3c7e21c2742, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=56e1b7b7-f18f-4875-9812-a3c7e21c2742, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=56e1b7b7-f18f-4875-9812-a3c7e21c2742, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=56e1b7b7-f18f-4875-9812-a3c7e21c2742, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.25.1.50:47100, err=Connection refused]]]] [12:41:03,374][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=56e1b7b7-f18f-4875-9812-a3c7e21c2742, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=23, intOrder=14, lastExchangeTime=1534239661663, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:03,375][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=25, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:41:03,375][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:03,375][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:03,375][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:03,375][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:03,376][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=25, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=56e1b7b7-f18f-4875-9812-a3c7e21c2742, customEvt=null, allowMerge=true] [12:41:03,376][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=25, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=25, minorTopVer=0], err=null] [12:41:03,377][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=25, minorTopVer=0], crd=false] [12:41:03,379][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=25, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=56e1b7b7-f18f-4875-9812-a3c7e21c2742, client=false] [12:41:03,525][INFO][grid-nio-worker-tcp-comm-7-#47][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45444] [12:41:03,526][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] Failed to connect to a remote node (make sure that destination node is alive and operating system firewall is disabled on local and remote hosts) [addrs=[/172.25.1.50:47101, /172.17.0.1:47101, /0:0:0:0:0:0:0:1%lo:47101, /127.0.0.1:47101]] [12:41:03,526][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] TcpCommunicationSpi failed to establish connection to node, node will be dropped from cluster [rmtNode=TcpDiscoveryNode [id=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=24, intOrder=15, lastExchangeTime=1534239661684, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true]] [12:41:03,527][INFO][tcp-comm-worker-#1][TcpDiscoverySpi] Pinging node: ba469bad-05bf-4954-ac68-bc4e9c61b4e2 [12:41:03,527][INFO][tcp-comm-worker-#1][TcpDiscoverySpi] Finished node ping [nodeId=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, res=false, time=3ms] [12:41:03,531][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=24, intOrder=15, lastExchangeTime=1534239661684, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:03,531][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=26, servers=4, clients=0, CPUs=32, offheap=75.0GB, heap=120.0GB] [12:41:03,531][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:03,531][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:03,531][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:03,531][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:03,532][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=26, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, customEvt=null, allowMerge=true] [12:41:03,533][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=26, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=26, minorTopVer=0], err=null] [12:41:03,533][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=26, minorTopVer=0], crd=false] [12:41:03,536][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=26, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=ba469bad-05bf-4954-ac68-bc4e9c61b4e2, client=false] [12:41:10,978][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery accepted incoming connection [rmtAddr=/172.25.1.50, rmtPort=51690] [12:41:10,978][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery spawning a new thread for connection [rmtAddr=/172.25.1.50, rmtPort=51690] [12:41:10,978][INFO][tcp-disco-sock-reader-#10][TcpDiscoverySpi] Started serving remote node connection [rmtAddr=/172.25.1.50:51690, rmtPort=51690] [12:41:11,125][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=928faf76-6465-445a-9ec9-aa4f0b1197a6, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=27, intOrder=16, lastExchangeTime=1534239671105, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:11,125][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=27, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:41:11,125][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:11,125][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:11,125][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:11,126][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:11,126][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=27, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=928faf76-6465-445a-9ec9-aa4f0b1197a6, customEvt=null, allowMerge=true] [12:41:11,128][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=27, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=27, minorTopVer=0], err=null] [12:41:11,129][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=27, minorTopVer=0], crd=false] [12:41:11,132][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=27, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=928faf76-6465-445a-9ec9-aa4f0b1197a6, client=false] [12:41:11,134][INFO][disco-event-worker-#61][GridDiscoveryManager] Added new node to topology: TcpDiscoveryNode [id=0698a26a-2805-4879-a76c-51846b2027ec, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=28, intOrder=17, lastExchangeTime=1534239671115, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:11,135][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=28, servers=4, clients=2, CPUs=48, offheap=110.0GB, heap=170.0GB] [12:41:11,135][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:11,135][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:11,135][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:11,135][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:11,136][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=28, minorTopVer=0], crd=false, evt=NODE_JOINED, evtNode=0698a26a-2805-4879-a76c-51846b2027ec, customEvt=null, allowMerge=true] [12:41:11,136][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=28, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=28, minorTopVer=0], err=null] [12:41:11,137][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=28, minorTopVer=0], crd=false] [12:41:11,139][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=28, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_JOINED, evtNode=0698a26a-2805-4879-a76c-51846b2027ec, client=false] [12:41:12,056][INFO][grid-nio-worker-tcp-comm-0-#40][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45448] [12:41:12,164][INFO][tcp-disco-sock-reader-#10][TcpDiscoverySpi] Finished serving remote node connection [rmtAddr=/172.25.1.50:51690, rmtPort=51690 [12:41:12,256][INFO][grid-nio-worker-tcp-comm-1-#41][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45450] [12:41:12,457][INFO][grid-nio-worker-tcp-comm-2-#42][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45452] [12:41:12,658][INFO][grid-nio-worker-tcp-comm-3-#43][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45454] [12:41:12,859][INFO][grid-nio-worker-tcp-comm-4-#44][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/172.17.0.1:47101, rmtAddr=/172.25.1.45:45456] [12:41:12,859][INFO][grid-nio-worker-tcp-comm-5-#45][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/0:0:0:0:0:0:0:1:47101, rmtAddr=/0:0:0:0:0:0:0:1:54800] [12:41:12,860][INFO][grid-nio-worker-tcp-comm-6-#46][TcpCommunicationSpi] Accepted incoming communication connection [locAddr=/127.0.0.1:47101, rmtAddr=/127.0.0.1:48944] [12:41:12,863][WARNING][tcp-disco-msg-worker-#2][TcpDiscoverySpi] Received EVT_NODE_FAILED event with warning [nodeInitiatedEvt=TcpDiscoveryNode [id=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.51], sockAddrs=[/172.25.1.51:47501, /0:0:0:0:0:0:0:1%lo:47501, /127.0.0.1:47501, /172.17.0.1:47501], discPort=47501, order=4, intOrder=4, lastExchangeTime=1534239629286, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false], msg=TcpCommunicationSpi failed to establish connection to node [rmtNode=TcpDiscoveryNode [id=0698a26a-2805-4879-a76c-51846b2027ec, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=28, intOrder=17, lastExchangeTime=1534239671109, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true], errs=class o.a.i.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=0698a26a-2805-4879-a76c-51846b2027ec, addrs=[/172.25.1.50:47101, /172.17.0.1:47101, /0:0:0:0:0:0:0:1%lo:47101, /127.0.0.1:47101]], connectErrs=[class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.25.1.50:47101, err=Connection refused], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=0698a26a-2805-4879-a76c-51846b2027ec, rcvd=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=0698a26a-2805-4879-a76c-51846b2027ec, rcvd=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=0698a26a-2805-4879-a76c-51846b2027ec, rcvd=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=0698a26a-2805-4879-a76c-51846b2027ec, rcvd=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47101, err=Remote node ID is not as expected [expected=0698a26a-2805-4879-a76c-51846b2027ec, rcvd=d5980f01-c881-4dcc-bb9b-0c5b53a9f2d4]]]]] [12:41:12,863][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=0698a26a-2805-4879-a76c-51846b2027ec, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=28, intOrder=17, lastExchangeTime=1534239671115, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:12,863][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=29, servers=4, clients=1, CPUs=48, offheap=94.0GB, heap=140.0GB] [12:41:12,863][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:12,863][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:12,863][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:12,864][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:12,864][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=0698a26a-2805-4879-a76c-51846b2027ec, customEvt=null, allowMerge=true] [12:41:12,865][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], err=null] [12:41:12,865][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], crd=false] [12:41:12,868][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=29, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=0698a26a-2805-4879-a76c-51846b2027ec, client=false] [12:41:13,665][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery accepted incoming connection [rmtAddr=/0:0:0:0:0:0:0:1, rmtPort=33137] [12:41:13,665][INFO][tcp-disco-srvr-#3][TcpDiscoverySpi] TCP discovery spawning a new thread for connection [rmtAddr=/0:0:0:0:0:0:0:1, rmtPort=33137] [12:41:13,665][INFO][tcp-disco-sock-reader-#12][TcpDiscoverySpi] Started serving remote node connection [rmtAddr=/0:0:0:0:0:0:0:1:33137, rmtPort=33137] [12:41:13,665][INFO][tcp-disco-sock-reader-#12][TcpDiscoverySpi] Received ping request from the remote node [rmtNodeId=784d8905-ba08-4a97-a3d5-99c3a33eccfb, rmtAddr=/0:0:0:0:0:0:0:1:33137, rmtPort=33137] [12:41:13,666][INFO][tcp-disco-sock-reader-#12][TcpDiscoverySpi] Finished writing ping response [rmtNodeId=784d8905-ba08-4a97-a3d5-99c3a33eccfb, rmtAddr=/0:0:0:0:0:0:0:1:33137, rmtPort=33137] [12:41:13,666][INFO][tcp-disco-sock-reader-#12][TcpDiscoverySpi] Finished serving remote node connection [rmtAddr=/0:0:0:0:0:0:0:1:33137, rmtPort=33137 [12:41:13,668][WARNING][tcp-disco-msg-worker-#2][TcpDiscoverySpi] Received EVT_NODE_FAILED event with warning [nodeInitiatedEvt=TcpDiscoveryNode [id=784d8905-ba08-4a97-a3d5-99c3a33eccfb, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.45], sockAddrs=[lab45.*.local/172.25.1.45:47500, /172.17.0.1:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1534239629226, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false], msg=TcpCommunicationSpi failed to establish connection to node [rmtNode=TcpDiscoveryNode [id=928faf76-6465-445a-9ec9-aa4f0b1197a6, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=27, intOrder=16, lastExchangeTime=1534239671004, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true], errs=class o.a.i.IgniteCheckedException: Failed to connect to node (is node still alive?). Make sure that each ComputeTask and cache Transaction has a timeout set in order to prevent parties from waiting forever in case of network issues [nodeId=928faf76-6465-445a-9ec9-aa4f0b1197a6, addrs=[/172.17.0.1:47100, /172.25.1.50:47100, /0:0:0:0:0:0:0:1%lo:47100, /127.0.0.1:47100]], connectErrs=[class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=928faf76-6465-445a-9ec9-aa4f0b1197a6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=928faf76-6465-445a-9ec9-aa4f0b1197a6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=928faf76-6465-445a-9ec9-aa4f0b1197a6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=928faf76-6465-445a-9ec9-aa4f0b1197a6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.17.0.1:47100, err=Remote node ID is not as expected [expected=928faf76-6465-445a-9ec9-aa4f0b1197a6, rcvd=784d8905-ba08-4a97-a3d5-99c3a33eccfb]], class o.a.i.IgniteCheckedException: Failed to connect to address [addr=/172.25.1.50:47100, err=Connection refused]]]] [12:41:13,669][WARNING][disco-event-worker-#61][GridDiscoveryManager] Node FAILED: TcpDiscoveryNode [id=928faf76-6465-445a-9ec9-aa4f0b1197a6, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.50], sockAddrs=[/172.17.0.1:0, /0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, /172.25.1.50:0], discPort=0, order=27, intOrder=16, lastExchangeTime=1534239671105, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=true] [12:41:13,669][INFO][disco-event-worker-#61][GridDiscoveryManager] Topology snapshot [ver=30, servers=4, clients=0, CPUs=32, offheap=75.0GB, heap=120.0GB] [12:41:13,669][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Node [id=23245A3B-3B69-4129-AE50-2D6AF3CD1232, clusterState=ACTIVE] [12:41:13,669][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- Baseline [id=0, size=4, online=4, offline=0] [12:41:13,669][INFO][disco-event-worker-#61][GridDiscoveryManager] Data Regions Configured: [12:41:13,669][INFO][disco-event-worker-#61][GridDiscoveryManager] ^-- default [initSize=256.0 MiB, maxSize=18.8 GiB, persistenceEnabled=true] [12:41:13,670][INFO][exchange-worker-#62][time] Started exchange init [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], crd=false, evt=NODE_FAILED, evtNode=928faf76-6465-445a-9ec9-aa4f0b1197a6, customEvt=null, allowMerge=true] [12:41:13,670][INFO][exchange-worker-#62][GridDhtPartitionsExchangeFuture] Finish exchange future [startVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], resVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], err=null] [12:41:13,671][INFO][exchange-worker-#62][time] Finished exchange init [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], crd=false] [12:41:13,674][INFO][exchange-worker-#62][GridCachePartitionExchangeManager] Skipping rebalancing (no affinity changes) [top=AffinityTopologyVersion [topVer=30, minorTopVer=0], rebTopVer=AffinityTopologyVersion [topVer=14, minorTopVer=0], evt=NODE_FAILED, evtNode=928faf76-6465-445a-9ec9-aa4f0b1197a6, client=false] [12:41:31,838][INFO][grid-timeout-worker-#39][IgniteKernal] Metrics for local node (to disable set 'metricsLogFrequency' to 0) ^-- Node [id=23245a3b, uptime=00:01:00.005] ^-- H/N/C [hosts=2, nodes=4, CPUs=32] ^-- CPU [cur=0.5%, avg=4.86%, GC=0%] ^-- PageMemory [pages=154469] ^-- Heap [used=1157MB, free=96.36%, comm=31744MB] ^-- Non heap [used=74MB, free=95.09%, comm=75MB] ^-- Outbound messages queue [size=0] ^-- Public thread pool [active=0, idle=16, qSize=0] ^-- System thread pool [active=0, idle=16, qSize=0] [12:42:31,840][INFO][grid-timeout-worker-#39][IgniteKernal] Metrics for local node (to disable set 'metricsLogFrequency' to 0) ^-- Node [id=23245a3b, uptime=00:02:00.014] ^-- H/N/C [hosts=2, nodes=4, CPUs=32] ^-- CPU [cur=0.07%, avg=2.38%, GC=0%] ^-- PageMemory [pages=154469] ^-- Heap [used=1205MB, free=96.2%, comm=31744MB] ^-- Non heap [used=76MB, free=94.98%, comm=77MB] ^-- Outbound messages queue [size=0] ^-- Public thread pool [active=0, idle=0, qSize=0] ^-- System thread pool [active=0, idle=6, qSize=0] [12:43:31,849][INFO][grid-timeout-worker-#39][IgniteKernal] Metrics for local node (to disable set 'metricsLogFrequency' to 0) ^-- Node [id=23245a3b, uptime=00:03:00.023] ^-- H/N/C [hosts=2, nodes=4, CPUs=32] ^-- CPU [cur=0%, avg=1.66%, GC=0%] ^-- PageMemory [pages=154469] ^-- Heap [used=1261MB, free=96.03%, comm=31744MB] ^-- Non heap [used=76MB, free=94.94%, comm=78MB] ^-- Outbound messages queue [size=0] ^-- Public thread pool [active=0, idle=0, qSize=0] ^-- System thread pool [active=0, idle=6, qSize=0] [12:43:38,302][INFO][db-checkpoint-thread-#66][GridCacheDatabaseSharedManager] Checkpoint started [checkpointId=39fd7827-edf9-4d23-8b42-a6b18bf82c16, startPtr=FileWALPointer [idx=22, fileOff=729367, len=41], checkpointLockWait=0ms, checkpointLockHoldTime=22ms, walCpRecordFsyncDuration=1ms, pages=31, reason='timeout'] [12:43:38,310][INFO][db-checkpoint-thread-#66][GridCacheDatabaseSharedManager] Checkpoint finished [cpId=39fd7827-edf9-4d23-8b42-a6b18bf82c16, pages=31, markPos=FileWALPointer [idx=22, fileOff=729367, len=41], walSegmentsCleared=0, walSegmentsCovered=[], markDuration=27ms, pagesWrite=1ms, fsync=6ms, total=34ms] [12:44:31,858][INFO][grid-timeout-worker-#39][IgniteKernal] Metrics for local node (to disable set 'metricsLogFrequency' to 0) ^-- Node [id=23245a3b, uptime=00:04:00.030] ^-- H/N/C [hosts=2, nodes=4, CPUs=32] ^-- CPU [cur=0.07%, avg=1.29%, GC=0%] ^-- PageMemory [pages=154469] ^-- Heap [used=1317MB, free=95.85%, comm=31744MB] ^-- Non heap [used=77MB, free=94.91%, comm=78MB] ^-- Outbound messages queue [size=0] ^-- Public thread pool [active=0, idle=0, qSize=0] ^-- System thread pool [active=0, idle=6, qSize=0] [12:45:15,092][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] Connect timed out (consider increasing 'failureDetectionTimeout' configuration property) [addr=/0:0:0:0:0:0:0:1%lo:47100, failureDetectionTimeout=10000] [12:45:15,092][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] Connect timed out (consider increasing 'failureDetectionTimeout' configuration property) [addr=/127.0.0.1:47100, failureDetectionTimeout=10000] [12:45:15,093][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] Connect timed out (consider increasing 'failureDetectionTimeout' configuration property) [addr=lab45.*.local/172.25.1.45:47100, failureDetectionTimeout=10000] [12:45:15,093][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] Connect timed out (consider increasing 'failureDetectionTimeout' configuration property) [addr=/172.17.0.1:47100, failureDetectionTimeout=10000] [12:45:15,093][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] Failed to connect to a remote node (make sure that destination node is alive and operating system firewall is disabled on local and remote hosts) [addrs=[/0:0:0:0:0:0:0:1%lo:47100, /127.0.0.1:47100, lab45.*.local/172.25.1.45:47100, /172.17.0.1:47100]] [12:45:15,093][WARNING][tcp-comm-worker-#1][TcpCommunicationSpi] TcpCommunicationSpi failed to establish connection to node, node will be dropped from cluster [rmtNode=TcpDiscoveryNode [id=784d8905-ba08-4a97-a3d5-99c3a33eccfb, addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.17.0.1, 172.25.1.45], sockAddrs=[lab45.*.local/172.25.1.45:47500, /172.17.0.1:47500, /0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1534239629226, loc=false, ver=2.5.1#20180807-sha1:8a4d3003, isClient=false]] [12:45:15,093][INFO][tcp-comm-worker-#1][TcpDiscoverySpi] Pinging node: 784d8905-ba08-4a97-a3d5-99c3a33eccfb [12:45:15,134][INFO][tcp-disco-sock-reader-#4][TcpDiscoverySpi] Finished serving remote node connection [rmtAddr=/172.25.1.51:46825, rmtPort=46825 bin/ignite.sh: line 181: 25630 Killed "$JAVA" ${JVM_OPTS} ${QUIET} "${RESTART_SUCCESS_OPT}" ${JMX_MON} -DIGNITE_HOME="${IGNITE_HOME}" -DIGNITE_PROG_NAME="$0" ${JVM_XOPTS} -cp "${CP}" ${MAIN_CLASS} "${CONFIG}"