[17:04:28,515][INFO][main][IgniteKernal] >>> __________ ________________ >>> / _/ ___/ |/ / _/_ __/ __/ >>> _/ // (7 7 // / / / / _/ >>> /___/\___/_/|_/___/ /_/ /___/ >>> >>> ver. 2.7.2#20190116-sha1:fc247e76 >>> 2018 Copyright(C) Apache Software Foundation >>> >>> Ignite documentation: http://ignite.apache.org [17:04:28,515][INFO][main][IgniteKernal] Config URL: file:/c:/Work/gridgain-ultimate-8.7.2/../1.xml [17:04:28,531][INFO][main][IgniteKernal] IgniteConfiguration [igniteInstanceName=null, pubPoolSize=8, svcPoolSize=8, callbackPoolSize=8, stripedPoolSize=8, sysPoolSize=8, mgmtPoolSize=4, igfsPoolSize=8, dataStreamerPoolSize=8, utilityCachePoolSize=8, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=8, igniteHome=c:\Work\gridgain-ultimate-8.7.2, igniteWorkDir=c:\Work\gridgain-ultimate-8.7.2\work, mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@6f94fa3e, nodeId=1cecc333-2890-40be-a581-ac374128501f, marsh=BinaryMarshaller [], marshLocJobs=false, daemon=false, p2pEnabled=true, netTimeout=5000, netCompressionLevel=1, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, sockTimeout=0, ackTimeout=0, marsh=null, reconCnt=10, reconDelay=2000, maxAckTimeout=600000, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null, skipAddrsRandomization=false], segPlc=STOP, segResolveAttempts=2, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=10000, commSpi=TcpCommunicationSpi [connectGate=null, connPlc=org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$FirstConnectionPolicy@4f67eb2a, enableForcibleNodeKill=false, enableTroubleshootingLog=false, locAddr=null, locHost=null, locPort=47100, locPortRange=100, shmemPort=-1, directBuf=true, directSndBuf=false, idleConnTimeout=600000, connTimeout=5000, maxConnTimeout=600000, reconCnt=10, sockSndBuf=32768, sockRcvBuf=32768, msgQueueLimit=0, slowClientQueueLimit=0, nioSrvr=null, shmemSrv=null, usePairedConnections=false, connectionsPerNode=1, tcpNoDelay=true, filterReachableAddresses=false, ackSndThreshold=32, unackedMsgsBufSize=0, sockWriteTimeout=2000, boundTcpPort=-1, boundTcpShmemPort=-1, selectorsCnt=4, selectorSpins=0, addrRslvr=null, ctxInitLatch=java.util.concurrent.CountDownLatch@4a668b6e[Count = 1], stopping=false], evtSpi=org.apache.ignite.spi.eventstorage.NoopEventStorageSpi@5f683daf, colSpi=NoopCollisionSpi [], deploySpi=LocalDeploymentSpi [], indexingSpi=org.apache.ignite.spi.indexing.noop.NoopIndexingSpi@6f27a732, addrRslvr=null, encryptionSpi=org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi@6c779568, clientMode=false, rebalanceThreadPoolSize=1, txCfg=TransactionConfiguration [txSerEnabled=false, dfltIsolation=REPEATABLE_READ, dfltConcurrency=PESSIMISTIC, dfltTxTimeout=0, txTimeoutOnPartitionMapExchange=0, pessimisticTxLogSize=0, pessimisticTxLogLinger=10000, tmLookupClsName=null, txManagerFactory=null, useJtaSync=false], cacheSanityCheckEnabled=true, discoStartupDelay=60000, deployMode=SHARED, p2pMissedCacheSize=100, locHost=null, timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=10000, sysWorkerBlockedTimeout=null, clientFailureDetectionTimeout=30000, metricsLogFreq=60000, hadoopCfg=null, connectorCfg=ConnectorConfiguration [jettyPath=null, host=null, port=11211, noDelay=true, directBuf=false, sndBufSize=32768, rcvBufSize=32768, idleQryCurTimeout=600000, idleQryCurCheckFreq=60000, sndQueueLimit=0, selectorCnt=4, idleTimeout=7000, sslEnabled=false, sslClientAuth=false, sslCtxFactory=null, sslFactory=null, portRange=100, threadPoolSize=8, msgInterceptor=null], odbcCfg=null, warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000, cacheMode=PARTITIONED, backups=1, aff=null, grpName=null], classLdr=null, sslCtxFactory=null, platformCfg=null, binaryCfg=null, memCfg=null, pstCfg=null, dsCfg=DataStorageConfiguration [sysRegionInitSize=41943040, sysRegionMaxSize=104857600, pageSize=0, concLvl=0, dfltDataRegConf=DataRegionConfiguration [name=Default_Region, maxSize=3410119065, initSize=104857600, swapPath=null, pageEvictionMode=DISABLED, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=true, checkpointPageBufSize=0], dataRegions=[DataRegionConfiguration [name=40MB_Region_Eviction, maxSize=41943040, initSize=20971520, swapPath=null, pageEvictionMode=RANDOM_2_LRU, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=false, checkpointPageBufSize=0], DataRegionConfiguration [name=30MB_Region_Swapping, maxSize=31457280, initSize=15728640, swapPath=dataRegionExampleSwap, pageEvictionMode=DISABLED, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=false, checkpointPageBufSize=0]], storagePath=null, checkpointFreq=180000, lockWaitTime=10000, checkpointThreads=4, checkpointWriteOrder=SEQUENTIAL, walHistSize=20, maxWalArchiveSize=1073741824, walSegments=10, walSegmentSize=67108864, walPath=db/wal, walArchivePath=db/wal/archive, metricsEnabled=false, walMode=LOG_ONLY, walTlbSize=131072, walBuffSize=0, walFlushFreq=2000, walFsyncDelay=1000, walRecordIterBuffSize=67108864, alwaysWriteFullPages=false, fileIOFactory=org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory@44a7bfbc, metricsSubIntervalCnt=5, metricsRateTimeInterval=60000, walAutoArchiveAfterInactivity=-1, writeThrottlingEnabled=false, walCompactionEnabled=false, walCompactionLevel=1, checkpointReadLockTimeout=null], activeOnStart=true, autoActivation=true, longQryWarnTimeout=3000, sqlConnCfg=null, cliConnCfg=ClientConnectorConfiguration [host=null, port=10800, portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true, maxOpenCursorsPerConn=128, threadPoolSize=8, idleTimeout=0, jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false, useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null], mvccVacuumThreadCnt=2, mvccVacuumFreq=5000, authEnabled=false, failureHnd=null, commFailureRslvr=null] [17:04:28,532][INFO][main][IgniteKernal] Daemon mode: off [17:04:28,532][INFO][main][IgniteKernal] OS: Windows 10 10.0 amd64 [17:04:28,532][INFO][main][IgniteKernal] OS user: gridgain [17:04:28,533][INFO][main][IgniteKernal] PID: 14516 [17:04:28,533][INFO][main][IgniteKernal] Language runtime: Java Platform API Specification ver. 1.8 [17:04:28,533][INFO][main][IgniteKernal] VM information: Java(TM) SE Runtime Environment 1.8.0_181-b13 Oracle Corporation Java HotSpot(TM) 64-Bit Server VM 25.181-b13 [17:04:28,534][INFO][main][IgniteKernal] VM total memory: 3.8GB [17:04:28,534][INFO][main][IgniteKernal] Remote Management [restart: on, REST: on, JMX (remote: on, port: 49720, auth: off, ssl: off)] [17:04:28,535][INFO][main][IgniteKernal] Logger: JavaLogger [quiet=false, config=null] [17:04:28,537][INFO][main][IgniteKernal] IGNITE_HOME=c:\Work\gridgain-ultimate-8.7.2 [17:04:28,538][INFO][main][IgniteKernal] VM arguments: [-XX:+AggressiveOpts, -Xms4g, -Xmx4g, -XX:MaxMetaspaceSize=256m, -DIGNITE_QUIET=false, -DIGNITE_SUCCESS_FILE=c:\Work\gridgain-ultimate-8.7.2\work\ignite_success_584ea4bf-3ed6-43b2-9133-c263674b7bee, -Dcom.sun.management.jmxremote, -Dcom.sun.management.jmxremote.port=49720, -Dcom.sun.management.jmxremote.authenticate=false, -Dcom.sun.management.jmxremote.ssl=false, -DIGNITE_HOME=c:\Work\gridgain-ultimate-8.7.2, -DIGNITE_PROG_NAME=ignite.bat] [17:04:28,538][INFO][main][IgniteKernal] System cache's DataRegion size is configured to 40 MB. Use DataStorageConfiguration.systemRegionInitialSize property to change the setting. [17:04:28,538][INFO][main][IgniteKernal] Configured caches [in 'sysMemPlc' dataRegion: ['ignite-sys-cache'], in 'Default_Region' dataRegion: ['cache_1', 'cache_2', 'cache_3', 'cache_4', 'cache_5', 'cache_6', 'cache_7', 'cache_8', 'cache_9', 'cache_10']] [17:04:28,538][WARNING][main][IgniteKernal] Peer class loading is enabled (disable it in production for performance and deployment consistency reasons) [17:04:28,539][WARNING][main][IgniteKernal] Please set system property '-Djava.net.preferIPv4Stack=true' to avoid possible problems in mixed environments. [17:04:28,540][WARNING][pub-#19][GridDiagnostic] This operating system has been tested less rigorously: Windows 10 10.0 amd64. Our team will appreciate the feedback if you experience any problems running ignite in this environment. [17:04:28,541][INFO][main][IgniteKernal] 3-rd party licenses can be found at: c:\Work\gridgain-ultimate-8.7.2\libs\licenses [17:04:28,743][INFO][main][IgnitePluginProcessor] Configured plugins: [17:04:28,744][INFO][main][IgnitePluginProcessor] ^-- GridGain 8.7.2#20190116-sha1:530463f0 [17:04:28,744][INFO][main][IgnitePluginProcessor] ^-- 2018 Copyright(C) GridGain Systems [17:04:28,744][INFO][main][IgnitePluginProcessor] [17:04:28,746][INFO][main][FailureProcessor] Configured failure handler: [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]]] [17:04:28,783][INFO][main][TcpCommunicationSpi] Successfully bound communication NIO server to TCP port [port=47100, locHost=0.0.0.0/0.0.0.0, selectorsCnt=4, selectorSpins=0, pairedConn=false] [17:04:28,834][WARNING][main][TcpCommunicationSpi] Message queue limit is set to 0 which may lead to potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and receiver sides. [17:04:28,846][WARNING][main][NoopCheckpointSpi] Checkpoints are disabled (to enable configure any GridCheckpointSpi implementation) [17:04:28,861][WARNING][main][GridCollisionManager] Collision resolution is disabled (all jobs will be activated upon arrival). [17:04:28,867][INFO][main][IgniteKernal] Security status [authentication=off, tls/ssl=off] [17:04:28,921][INFO][main][TcpDiscoverySpi] Successfully bound to TCP port [port=47500, localHost=0.0.0.0/0.0.0.0, locNodeId=1cecc333-2890-40be-a581-ac374128501f] [17:04:28,936][INFO][main][PdsFoldersResolver] Successfully created new persistent storage folder [c:\Work\gridgain-ultimate-8.7.2\work\db\node00-63589419-0434-4b6a-b0df-edcfde149042] [17:04:28,936][INFO][main][PdsFoldersResolver] Consistent ID used for local node is [63589419-0434-4b6a-b0df-edcfde149042] according to persistence data storage folders [17:04:28,938][INFO][main][CacheObjectBinaryProcessorImpl] Resolved directory for serialized binary metadata: c:\Work\gridgain-ultimate-8.7.2\work\binary_meta\node00-63589419-0434-4b6a-b0df-edcfde149042 [17:04:29,089][INFO][main][FilePageStoreManager] Resolved page store work directory: c:\Work\gridgain-ultimate-8.7.2\work\db\node00-63589419-0434-4b6a-b0df-edcfde149042 [17:04:29,095][INFO][main][FileWriteAheadLogManager] Resolved write ahead log work directory: c:\Work\gridgain-ultimate-8.7.2\work\db\wal\node00-63589419-0434-4b6a-b0df-edcfde149042 [17:04:29,097][INFO][main][FileWriteAheadLogManager] Resolved write ahead log archive directory: c:\Work\gridgain-ultimate-8.7.2\work\db\wal\archive\node00-63589419-0434-4b6a-b0df-edcfde149042 [17:04:29,366][INFO][main][FileHandleManagerImpl] Started write-ahead log manager [mode=LOG_ONLY] [17:04:29,372][WARNING][main][GridCacheDatabaseSharedManager] DataRegionConfiguration.maxWalArchiveSize instead DataRegionConfiguration.walHistorySize would be used for removing old archive wal files [17:04:29,391][INFO][main][GridCacheDatabaseSharedManager] Configured data regions initialized successfully [total=6] [17:04:29,411][INFO][main][PartitionsEvictManager] Evict partition permits=2 [17:04:29,549][INFO][main][ClientListenerProcessor] Client connector processor has started on TCP port 10800 [17:04:29,584][INFO][main][GridTcpRestProtocol] Command protocol successfully started [name=TCP binary, host=0.0.0.0/0.0.0.0, port=11211] [17:04:29,614][WARNING][main][GridPluginProvider] Rolling updates are disabled. GridGain version update will require full cluster restart. Consider changing 'GridGainConfiguration.rollingUpdatesEnabled' configuration property. [17:04:29,618][INFO][main][GridPluginProvider] GridGain Configuration: GridGainConfiguration [dataCenterId=0, interopCfg=null, drRcvCfg=null, drSndCfg=null, snapshotCfg=null, licUrl=null, authenticator=null, securityCred=null, rollingUpdatesEnabled=false, drPoolSize=4, drStreamerPoolSize=8, drUseCacheNames=false] [17:04:29,730][INFO][main][IgniteKernal] Non-loopback local IPs: 172.25.4.52, fe80:0:0:0:2d2b:5bfb:e052:9fb0%eth5 [17:04:29,730][INFO][main][IgniteKernal] Enabled local MACs: 54A050520B61 [17:04:29,735][INFO][main][GridCacheDatabaseSharedManager] Read checkpoint status [startMarker=null, endMarker=null] [17:04:29,741][INFO][main][PageMemoryImpl] Started page memory [memoryAllocated=100,0 MiB, pages=24808, tableSize=1,9 MiB, checkpointBuffer=100,0 MiB] [17:04:29,742][INFO][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=FileWALPointer [idx=0, fileOff=0, len=0], lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [17:04:29,768][INFO][main][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [17:04:29,772][INFO][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=0 ms] [17:04:29,772][INFO][main][GridCacheProcessor] Restoring partition state for local groups. [17:04:29,772][INFO][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=0, partitionsProcessed=0, time=0ms] [17:04:29,850][INFO][main][FilePageStoreManager] Cleanup cache stores [total=1, left=0, cleanFiles=false] [17:04:29,865][INFO][main][PageMemoryImpl] Started page memory [memoryAllocated=100,0 MiB, pages=24808, tableSize=1,9 MiB, checkpointBuffer=100,0 MiB] [17:04:29,866][INFO][main][PageMemoryImpl] Started page memory [memoryAllocated=100,0 MiB, pages=24808, tableSize=1,9 MiB, checkpointBuffer=100,0 MiB] [17:04:29,868][INFO][main][PageMemoryImpl] Started page memory [memoryAllocated=100,0 MiB, pages=24808, tableSize=1,9 MiB, checkpointBuffer=100,0 MiB] [17:04:29,923][INFO][main][PageMemoryImpl] Started page memory [memoryAllocated=3,2 GiB, pages=807016, tableSize=62,8 MiB, checkpointBuffer=813,0 MiB] [17:04:29,930][INFO][main][GridCacheDatabaseSharedManager] Configured data regions started successfully [total=6] [17:04:29,931][INFO][main][GridCacheDatabaseSharedManager] Starting binary memory restore for: [-2100569601, 540205750, 540205751, -433490948, 540205748, 540205749, 540205754, 540205755, 540205752, 540205753, 540205756] [17:04:38,587][INFO][main][GridCacheDatabaseSharedManager] Read checkpoint status [startMarker=null, endMarker=null] [17:04:38,587][INFO][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=FileWALPointer [idx=0, fileOff=0, len=0], lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [17:04:38,594][INFO][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=c:\Work\gridgain-ultimate-8.7.2\work\db\wal\node00-63589419-0434-4b6a-b0df-edcfde149042\0000000000000000.wal, offset=0, ver=2] [17:04:38,698][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=ignite-sys-cache, id=-2100569601, dataRegionName=sysMemPlc, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647, mvcc=false] [17:04:38,722][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_3, id=540205750, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,727][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_4, id=540205751, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,733][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_10, id=-433490948, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,738][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_1, id=540205748, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,743][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_2, id=540205749, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,748][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_7, id=540205754, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,753][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_8, id=540205755, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,758][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_5, id=540205752, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,762][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_6, id=540205753, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,767][INFO][main][GridCacheProcessor] Started cache in recovery mode [name=cache_9, id=540205756, dataRegionName=Default_Region, mode=PARTITIONED, atomicity=ATOMIC, backups=1, mvcc=false] [17:04:38,770][INFO][main][GridCacheDatabaseSharedManager] Binary recovery performed in 8839 ms. [17:04:38,772][INFO][main][GridCacheDatabaseSharedManager] Partition states information after BINARY RECOVERY phase: [17:04:45,180][INFO][main][GridCacheDatabaseSharedManager] Read checkpoint status [startMarker=null, endMarker=null] [17:04:45,181][INFO][main][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=FileWALPointer [idx=0, fileOff=0, len=0], lastCheckpointId=00000000-0000-0000-0000-000000000000] [17:04:45,248][INFO][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=65 ms] [17:04:45,248][INFO][main][GridCacheProcessor] Restoring partition state for local groups. [17:04:51,386][INFO][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=11, partitionsProcessed=0, time=6135ms] [17:04:51,387][INFO][main][GridCacheDatabaseSharedManager] Partition states information after LOGICAL RECOVERY phase: [17:04:54,847][SEVERE][nio-acceptor-tcp-comm-#28][G] Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [threadName=ttl-cleanup-worker, blockedFor=16s] [17:04:54,848][WARNING][nio-acceptor-tcp-comm-#28][G] Thread [name="ttl-cleanup-worker-#45", id=74, state=WAITING, blockCnt=0, waitCnt=1] [17:04:54,849][SEVERE][nio-acceptor-tcp-comm-#28][] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=ttl-cleanup-worker, igniteInstanceName=null, finished=false, heartbeatTs=1548252278711]]] class org.apache.ignite.IgniteException: GridWorker [name=ttl-cleanup-worker, igniteInstanceName=null, finished=false, heartbeatTs=1548252278711] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1833) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1828) at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233) at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) at org.apache.ignite.internal.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:2938) at org.apache.ignite.internal.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2877) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) [17:04:54,849][WARNING][nio-acceptor-tcp-comm-#28][FailureProcessor] No deadlocked threads detected. [17:04:55,030][WARNING][nio-acceptor-tcp-comm-#28][FailureProcessor] Thread dump at 2019/01/23 17:04:55 MSK Thread [name="ttl-cleanup-worker-#45", id=74, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at o.a.i.i.managers.discovery.GridDiscoveryManager.localJoin(GridDiscoveryManager.java:2205) at o.a.i.i.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:133) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="wal-segment-syncer-#44", id=73, state=TIMED_WAITING, blockCnt=0, waitCnt=33] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:7632) at o.a.i.i.processors.cache.persistence.wal.filehandle.FileHandleManagerImpl$WalSegmentSyncer.body(FileHandleManagerImpl.java:582) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="wal-file-archiver%null-#43", id=72, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=o.a.i.i.processors.cache.persistence.wal.aware.SegmentCurrentStateStorage@6d9e7301, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentCurrentStateStorage.awaitSegment(SegmentCurrentStateStorage.java:73) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentCurrentStateStorage.waitNextSegmentForArchivation(SegmentCurrentStateStorage.java:90) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentAware.waitNextSegmentForArchivation(SegmentAware.java:79) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver.body(FileWriteAheadLogManager.java:1661) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-9", id=71, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@600b0b7 Thread [name="Thread-8", id=70, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5be82d43 Thread [name="Thread-7", id=69, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@82c57b3 Thread [name="Thread-6", id=68, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@4fdfa676 Thread [name="Thread-5", id=67, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@3aacf32a Thread [name="Thread-4", id=66, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6ab72419 Thread [name="Thread-3", id=65, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@7957dc72 Thread [name="Thread-2", id=64, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@70e0accd Thread [name="Thread-1", id=63, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-data-loader-flusher-#42", id=62, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@26f33dc1, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.DelayQueue.take(DelayQueue.java:211) at o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java:109) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-3-#40", id=60, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3c27379c - locked java.util.Collections$UnmodifiableSet@1c4d72aa - locked sun.nio.ch.WindowsSelectorImpl@2d50c59 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-2-#39", id=59, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3df734f7 - locked java.util.Collections$UnmodifiableSet@38696f02 - locked sun.nio.ch.WindowsSelectorImpl@4fc68287 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-1-#38", id=58, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@655be8e2 - locked java.util.Collections$UnmodifiableSet@610f2832 - locked sun.nio.ch.WindowsSelectorImpl@12ed5a6e at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-0-#37", id=57, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@69c4300e - locked java.util.Collections$UnmodifiableSet@53e2c92c - locked sun.nio.ch.WindowsSelectorImpl@4b3425da at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-tcp-rest-#41", id=61, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@337238f8 - locked java.util.Collections$UnmodifiableSet@7bfe3ad8 - locked sun.nio.ch.WindowsSelectorImpl@3e902722 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:2929) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2877) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-3-#34", id=54, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@5460cfdf - locked java.util.Collections$UnmodifiableSet@76779853 - locked sun.nio.ch.WindowsSelectorImpl@7c5544bf at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-2-#33", id=53, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4343d520 - locked java.util.Collections$UnmodifiableSet@118fe4a0 - locked sun.nio.ch.WindowsSelectorImpl@5bf053c2 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-1-#32", id=52, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@d5684bf - locked java.util.Collections$UnmodifiableSet@5061bae0 - locked sun.nio.ch.WindowsSelectorImpl@3ea5eea8 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-0-#31", id=51, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6792d4fd - locked java.util.Collections$UnmodifiableSet@5e58c7cc - locked sun.nio.ch.WindowsSelectorImpl@333c05d at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-client-listener-#35", id=55, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@1272ae3d - locked java.util.Collections$UnmodifiableSet@6336a5b2 - locked sun.nio.ch.WindowsSelectorImpl@7bc868bf at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:2929) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2877) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="db-snapshot-msg-executor-#30", id=50, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager.lambda$startSnapshotExecutors$0(GridCacheSnapshotManager.java:448) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager$$Lambda$23/179294202.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@235a0c16 Thread [name="db-snapshot-executor-#29", id=49, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager.lambda$startSnapshotExecutors$0(GridCacheSnapshotManager.java:448) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager$$Lambda$23/179294202.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6955cb39 Thread [name="tcp-comm-worker-#1", id=48, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@66e46eb3, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at o.a.i.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.body(TcpCommunicationSpi.java:4298) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at o.a.i.spi.communication.tcp.TcpCommunicationSpi$5.body(TcpCommunicationSpi.java:2240) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:62) Thread [name="grid-nio-worker-tcp-comm-3-#27", id=43, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@61603714 - locked java.util.Collections$UnmodifiableSet@37ad15bf - locked sun.nio.ch.WindowsSelectorImpl@f981e65 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-2-#26", id=42, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4acc78d3 - locked java.util.Collections$UnmodifiableSet@2a742e09 - locked sun.nio.ch.WindowsSelectorImpl@75abd615 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-1-#25", id=41, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3a0f85f3 - locked java.util.Collections$UnmodifiableSet@13ac7881 - locked sun.nio.ch.WindowsSelectorImpl@18a3189b at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-0-#24", id=40, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@33937ee4 - locked java.util.Collections$UnmodifiableSet@75754578 - locked sun.nio.ch.WindowsSelectorImpl@56d4327 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-tcp-comm-#28", id=47, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.management.ThreadImpl.dumpThreads0(Native Method) at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:454) at o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1367) at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:136) - locked o.a.i.i.processors.failure.FailureProcessor@50af4d1 at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:112) at o.a.i.i.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831) at o.a.i.i.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1828) at o.a.i.i.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233) at o.a.i.i.util.worker.GridWorker.onIdle(GridWorker.java:297) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:2938) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2877) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-timeout-worker-#23", id=39, state=TIMED_WAITING, blockCnt=2, waitCnt=81] Lock [object=java.lang.Object@2dc6deca, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:269) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="ignite-clock", id=38, state=TIMED_WAITING, blockCnt=0, waitCnt=2433] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils$22.run(IgniteUtils.java:3382) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#22", id=37, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#21", id=36, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#20", id=35, state=TIMED_WAITING, blockCnt=1, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#19", id=34, state=TIMED_WAITING, blockCnt=1, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#18", id=33, state=TIMED_WAITING, blockCnt=1, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#17", id=32, state=TIMED_WAITING, blockCnt=3, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="jvm-pause-detector-worker", id=31, state=TIMED_WAITING, blockCnt=0, waitCnt=519] at java.lang.Thread.sleep(Native Method) at o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:100) Thread [name="data-streamer-stripe-7-#16", id=30, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-6-#15", id=29, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-5-#14", id=28, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-4-#13", id=27, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-3-#12", id=26, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-2-#11", id=25, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-1-#10", id=24, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-0-#9", id=23, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-7-#8", id=22, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-6-#7", id=21, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-5-#6", id=20, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-4-#5", id=19, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-3-#4", id=18, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-2-#3", id=17, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-1-#2", id=16, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-0-#1", id=15, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="RMI TCP Accept-0", id=14, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.DualStackPlainSocketImpl.accept0(Native Method) at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199) - locked java.net.SocksSocketImpl@1d5c018b at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Thread [name="RMI TCP Accept-49720", id=13, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.DualStackPlainSocketImpl.accept0(Native Method) at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199) - locked java.net.SocksSocketImpl@4f22c28d at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Thread [name="RMI TCP Accept-0", id=12, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.DualStackPlainSocketImpl.accept0(Native Method) at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199) - locked java.net.SocksSocketImpl@5409752d at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Thread [name="Attach Listener", id=5, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Signal Dispatcher", id=4, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Finalizer", id=3, state=WAITING, blockCnt=44, waitCnt=4] Lock [object=java.lang.ref.ReferenceQueue$Lock@1e80c45b, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Thread [name="Reference Handler", id=2, state=WAITING, blockCnt=3, waitCnt=3] Lock [object=java.lang.ref.Reference$Lock@63adce59, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Thread [name="main", id=1, state=RUNNABLE, blockCnt=3, waitCnt=32] at java.io.WinNTFileSystem.getBooleanAttributes(Native Method) at java.io.File.exists(File.java:819) at o.a.i.i.processors.cache.persistence.file.FilePageStore.exists(FilePageStore.java:120) at o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.exists(FilePageStoreManager.java:486) at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.dumpPartitionsInfo(GridCacheDatabaseSharedManager.java:4673) at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.dumpPartitionsInfo(GridCacheDatabaseSharedManager.java:4641) at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.startMemoryRestore(GridCacheDatabaseSharedManager.java:1955) at o.a.i.i.IgniteKernal.start(IgniteKernal.java:1050) at o.a.i.i.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:2040) at o.a.i.i.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1732) - locked o.a.i.i.IgnitionEx$IgniteNamedInstance@3ac25e0b at o.a.i.i.IgnitionEx.start0(IgnitionEx.java:1158) at o.a.i.i.IgnitionEx.startConfigurations(IgnitionEx.java:1076) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:962) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:861) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:731) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:700) at o.a.i.Ignition.start(Ignition.java:348) at o.a.i.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:301) [17:04:57,812][INFO][main][TcpDiscoverySpi] Connection check threshold is calculated: 10000 [17:05:00,855][SEVERE][tcp-disco-msg-worker-#2][G] Blocked system-critical thread has been detected. This can lead to cluster-wide undefined behaviour [threadName=ttl-cleanup-worker, blockedFor=22s] [17:05:00,856][WARNING][tcp-disco-msg-worker-#2][G] Thread [name="ttl-cleanup-worker-#45", id=74, state=WAITING, blockCnt=0, waitCnt=1] [17:05:00,859][SEVERE][tcp-disco-msg-worker-#2][] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=[SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_WORKER_BLOCKED, err=class o.a.i.IgniteException: GridWorker [name=ttl-cleanup-worker, igniteInstanceName=null, finished=false, heartbeatTs=1548252278711]]] class org.apache.ignite.IgniteException: GridWorker [name=ttl-cleanup-worker, igniteInstanceName=null, finished=false, heartbeatTs=1548252278711] at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1833) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1828) at org.apache.ignite.internal.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233) at org.apache.ignite.internal.util.worker.GridWorker.onIdle(GridWorker.java:297) at org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2680) at org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7221) at org.apache.ignite.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2717) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at org.apache.ignite.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7159) at org.apache.ignite.spi.IgniteSpiThread.run(IgniteSpiThread.java:62) [17:05:00,862][WARNING][tcp-disco-msg-worker-#2][FailureProcessor] No deadlocked threads detected. [17:05:01,078][WARNING][tcp-disco-msg-worker-#2][FailureProcessor] Thread dump at 2019/01/23 17:05:01 MSK Thread [name="tcp-disco-multicast-addr-sender-#4", id=78, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.TwoStacksPlainDatagramSocketImpl.receive0(Native Method) - locked java.net.TwoStacksPlainDatagramSocketImpl@3bd8dd3c at java.net.TwoStacksPlainDatagramSocketImpl.receive(TwoStacksPlainDatagramSocketImpl.java:114) - locked java.net.TwoStacksPlainDatagramSocketImpl@3bd8dd3c at java.net.DatagramSocket.receive(DatagramSocket.java:812) - locked java.net.DatagramPacket@3fe74e9f - locked java.net.MulticastSocket@7ae5d707 at o.a.i.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder$AddressSender.body(TcpDiscoveryMulticastIpFinder.java:882) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:62) Thread [name="tcp-disco-srvr-#3", id=77, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.DualStackPlainSocketImpl.accept0(Native Method) at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199) - locked java.net.SocksSocketImpl@3619a5e9 at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at o.a.i.spi.discovery.tcp.ServerImpl$TcpServer.body(ServerImpl.java:5885) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at o.a.i.spi.discovery.tcp.ServerImpl$TcpServerThread.body(ServerImpl.java:5803) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:62) Thread [name="tcp-disco-msg-worker-#2", id=76, state=RUNNABLE, blockCnt=0, waitCnt=340] at sun.management.ThreadImpl.dumpThreads0(Native Method) at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:454) at o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1367) at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:136) - locked o.a.i.i.processors.failure.FailureProcessor@50af4d1 at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:112) at o.a.i.i.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1831) at o.a.i.i.IgnitionEx$IgniteNamedInstance$2.apply(IgnitionEx.java:1828) at o.a.i.i.worker.WorkersRegistry.onIdle(WorkersRegistry.java:233) at o.a.i.i.util.worker.GridWorker.onIdle(GridWorker.java:297) at o.a.i.spi.discovery.tcp.ServerImpl$RingMessageWorker.lambda$new$0(ServerImpl.java:2680) at o.a.i.spi.discovery.tcp.ServerImpl$RingMessageWorker$$Lambda$72/1943444495.run(Unknown Source) at o.a.i.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:7221) at o.a.i.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:2717) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at o.a.i.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:7159) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:62) Thread [name="disco-notifier-worker-#46", id=75, state=WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4b55af2b, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2675) at o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2720) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="ttl-cleanup-worker-#45", id=74, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at o.a.i.i.managers.discovery.GridDiscoveryManager.localJoin(GridDiscoveryManager.java:2205) at o.a.i.i.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:133) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-9", id=71, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@600b0b7 Thread [name="Thread-8", id=70, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5be82d43 Thread [name="Thread-7", id=69, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@82c57b3 Thread [name="Thread-6", id=68, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@4fdfa676 Thread [name="Thread-5", id=67, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@3aacf32a Thread [name="Thread-4", id=66, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6ab72419 Thread [name="Thread-3", id=65, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@7957dc72 Thread [name="Thread-2", id=64, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@70e0accd Thread [name="Thread-1", id=63, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.Iocp.getQueuedCompletionStatus(Native Method) at sun.nio.ch.Iocp.access$300(Iocp.java:46) at sun.nio.ch.Iocp$EventHandlerTask.run(Iocp.java:333) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-data-loader-flusher-#42", id=62, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@26f33dc1, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.DelayQueue.take(DelayQueue.java:211) at o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java:109) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-3-#40", id=60, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3c27379c - locked java.util.Collections$UnmodifiableSet@1c4d72aa - locked sun.nio.ch.WindowsSelectorImpl@2d50c59 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-2-#39", id=59, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3df734f7 - locked java.util.Collections$UnmodifiableSet@38696f02 - locked sun.nio.ch.WindowsSelectorImpl@4fc68287 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-1-#38", id=58, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@655be8e2 - locked java.util.Collections$UnmodifiableSet@610f2832 - locked sun.nio.ch.WindowsSelectorImpl@12ed5a6e at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-0-#37", id=57, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@69c4300e - locked java.util.Collections$UnmodifiableSet@53e2c92c - locked sun.nio.ch.WindowsSelectorImpl@4b3425da at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-tcp-rest-#41", id=61, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@337238f8 - locked java.util.Collections$UnmodifiableSet@7bfe3ad8 - locked sun.nio.ch.WindowsSelectorImpl@3e902722 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:2929) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2877) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-3-#34", id=54, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@5460cfdf - locked java.util.Collections$UnmodifiableSet@76779853 - locked sun.nio.ch.WindowsSelectorImpl@7c5544bf at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-2-#33", id=53, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4343d520 - locked java.util.Collections$UnmodifiableSet@118fe4a0 - locked sun.nio.ch.WindowsSelectorImpl@5bf053c2 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-1-#32", id=52, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@d5684bf - locked java.util.Collections$UnmodifiableSet@5061bae0 - locked sun.nio.ch.WindowsSelectorImpl@3ea5eea8 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-0-#31", id=51, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6792d4fd - locked java.util.Collections$UnmodifiableSet@5e58c7cc - locked sun.nio.ch.WindowsSelectorImpl@333c05d at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-client-listener-#35", id=55, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@1272ae3d - locked java.util.Collections$UnmodifiableSet@6336a5b2 - locked sun.nio.ch.WindowsSelectorImpl@7bc868bf at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:2929) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2877) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="db-snapshot-msg-executor-#30", id=50, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager.lambda$startSnapshotExecutors$0(GridCacheSnapshotManager.java:448) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager$$Lambda$23/179294202.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@235a0c16 Thread [name="db-snapshot-executor-#29", id=49, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager.lambda$startSnapshotExecutors$0(GridCacheSnapshotManager.java:448) at org.gridgain.grid.internal.processors.cache.database.snapshot.GridCacheSnapshotManager$$Lambda$23/179294202.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6955cb39 Thread [name="tcp-comm-worker-#1", id=48, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@66e46eb3, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at o.a.i.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.body(TcpCommunicationSpi.java:4298) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at o.a.i.spi.communication.tcp.TcpCommunicationSpi$5.body(TcpCommunicationSpi.java:2240) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:62) Thread [name="grid-nio-worker-tcp-comm-3-#27", id=43, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@61603714 - locked java.util.Collections$UnmodifiableSet@37ad15bf - locked sun.nio.ch.WindowsSelectorImpl@f981e65 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-2-#26", id=42, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4acc78d3 - locked java.util.Collections$UnmodifiableSet@2a742e09 - locked sun.nio.ch.WindowsSelectorImpl@75abd615 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-1-#25", id=41, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3a0f85f3 - locked java.util.Collections$UnmodifiableSet@13ac7881 - locked sun.nio.ch.WindowsSelectorImpl@18a3189b at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-0-#24", id=40, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@33937ee4 - locked java.util.Collections$UnmodifiableSet@75754578 - locked sun.nio.ch.WindowsSelectorImpl@56d4327 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2151) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1797) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-tcp-comm-#28", id=47, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method) at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296) at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278) at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@4e438368 - locked java.util.Collections$UnmodifiableSet@1aca2848 - locked sun.nio.ch.WindowsSelectorImpl@6107f015 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:2929) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2877) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-timeout-worker-#23", id=39, state=TIMED_WAITING, blockCnt=2, waitCnt=106] Lock [object=java.lang.Object@2dc6deca, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:269) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="ignite-clock", id=38, state=TIMED_WAITING, blockCnt=0, waitCnt=2977] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils$22.run(IgniteUtils.java:3382) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#22", id=37, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#21", id=36, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#20", id=35, state=TIMED_WAITING, blockCnt=1, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#19", id=34, state=TIMED_WAITING, blockCnt=1, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#18", id=33, state=TIMED_WAITING, blockCnt=1, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#17", id=32, state=TIMED_WAITING, blockCnt=3, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72c7008f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="jvm-pause-detector-worker", id=31, state=TIMED_WAITING, blockCnt=0, waitCnt=635] at java.lang.Thread.sleep(Native Method) at o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:100) Thread [name="data-streamer-stripe-7-#16", id=30, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-6-#15", id=29, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-5-#14", id=28, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-4-#13", id=27, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-3-#12", id=26, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-2-#11", id=25, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-1-#10", id=24, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-0-#9", id=23, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-7-#8", id=22, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-6-#7", id=21, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-5-#6", id=20, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-4-#5", id=19, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-3-#4", id=18, state=WAITING, blockCnt=1, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-2-#3", id=17, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-1-#2", id=16, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-0-#1", id=15, state=WAITING, blockCnt=2, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:669) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:493) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Thread [name="RMI TCP Accept-0", id=14, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.DualStackPlainSocketImpl.accept0(Native Method) at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199) - locked java.net.SocksSocketImpl@1d5c018b at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Thread [name="RMI TCP Accept-49720", id=13, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.DualStackPlainSocketImpl.accept0(Native Method) at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199) - locked java.net.SocksSocketImpl@4f22c28d at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Thread [name="RMI TCP Accept-0", id=12, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.DualStackPlainSocketImpl.accept0(Native Method) at java.net.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:131) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:199) - locked java.net.SocksSocketImpl@5409752d at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Thread [name="Attach Listener", id=5, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Signal Dispatcher", id=4, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Finalizer", id=3, state=WAITING, blockCnt=44, waitCnt=4] Lock [object=java.lang.ref.ReferenceQueue$Lock@1e80c45b, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Thread [name="Reference Handler", id=2, state=WAITING, blockCnt=3, waitCnt=3] Lock [object=java.lang.ref.Reference$Lock@63adce59, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Thread [name="main", id=1, state=RUNNABLE, blockCnt=5, waitCnt=35] at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method) at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) - locked java.net.DualStackPlainSocketImpl@7cd37e9 at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) at java.net.Socket.connect(Socket.java:589) at o.a.i.spi.discovery.tcp.TcpDiscoverySpi.openSocket(TcpDiscoverySpi.java:1512) at o.a.i.spi.discovery.tcp.TcpDiscoverySpi.openSocket(TcpDiscoverySpi.java:1474) at o.a.i.spi.discovery.tcp.ServerImpl.sendMessageDirectly(ServerImpl.java:1222) at o.a.i.spi.discovery.tcp.ServerImpl.sendJoinRequestMessage(ServerImpl.java:1069) at o.a.i.spi.discovery.tcp.ServerImpl.joinTopology(ServerImpl.java:913) at o.a.i.spi.discovery.tcp.ServerImpl.spiStart(ServerImpl.java:394) at o.a.i.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:2023) at o.a.i.i.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:297) at o.a.i.i.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:935) at o.a.i.i.IgniteKernal.startManager(IgniteKernal.java:1690) at o.a.i.i.IgniteKernal.start(IgniteKernal.java:1070) at o.a.i.i.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:2040) at o.a.i.i.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1732) - locked o.a.i.i.IgnitionEx$IgniteNamedInstance@3ac25e0b at o.a.i.i.IgnitionEx.start0(IgnitionEx.java:1158) at o.a.i.i.IgnitionEx.startConfigurations(IgnitionEx.java:1076) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:962) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:861) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:731) at o.a.i.i.IgnitionEx.start(IgnitionEx.java:700) at o.a.i.Ignition.start(Ignition.java:348) at o.a.i.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:301) Locked synchronizers: java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@5547acbb java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@1e845633 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@40dd9aae java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2ec19f23 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@460bcc34 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@7a822f08 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@5626c21c java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@743c677c [17:05:08,668][INFO][disco-notifier-worker-#46][MvccProcessorImpl] Assigned mvcc coordinator [crd=MvccCoordinator [nodeId=1cecc333-2890-40be-a581-ac374128501f, crdVer=1548252308644, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0]], crdNode=TcpDiscoveryNode [id=1cecc333-2890-40be-a581-ac374128501f, addrs=[0:0:0:0:0:0:0:1, 127.0.0.1, 172.25.4.52], sockAddrs=[work-pc.gridgain.local/172.25.4.52:47500, /0:0:0:0:0:0:0:1:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1548252297804, loc=true, ver=2.7.2#20190116-sha1:fc247e76, isClient=false]] [17:05:08,668][INFO][disco-notifier-worker-#46][MvccProcessorImpl] Initialize local node as mvcc coordinator [node=1cecc333-2890-40be-a581-ac374128501f, crdVer=1548252308644] [17:05:08,724][INFO][main][IgniteKernal] Performance suggestions for grid (fix if possible) [17:05:08,724][INFO][main][IgniteKernal] To disable, set -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true [17:05:08,725][INFO][main][IgniteKernal] ^-- Decrease number of backups (set 'backups' to 0) [17:05:08,725][INFO][main][IgniteKernal] ^-- Enable G1 Garbage Collector (add '-XX:+UseG1GC' to JVM options) [17:05:08,725][INFO][main][IgniteKernal] ^-- Set max direct memory size if getting 'OOME: Direct buffer memory' (add '-XX:MaxDirectMemorySize=[g|G|m|M|k|K]' to JVM options) [17:05:08,725][INFO][main][IgniteKernal] ^-- Disable processing of calls to System.gc() (add '-XX:+DisableExplicitGC' to JVM options) [17:05:08,725][INFO][main][IgniteKernal] Refer to this page for more performance suggestions: https://apacheignite.readme.io/docs/jvm-and-system-tuning [17:05:08,725][INFO][main][IgniteKernal] [17:05:08,725][INFO][main][IgniteKernal] To start Console Management & Monitoring run ignitevisorcmd.{sh|bat} [17:05:08,727][INFO][main][IgniteKernal] Data Regions Configured: [17:05:08,727][INFO][main][IgniteKernal] ^-- Default_Region [initSize=100,0 MiB, maxSize=3,2 GiB, persistence=true] [17:05:08,728][INFO][main][IgniteKernal] ^-- 40MB_Region_Eviction [initSize=20,0 MiB, maxSize=40,0 MiB, persistence=false] [17:05:08,728][INFO][main][IgniteKernal] ^-- 30MB_Region_Swapping [initSize=15,0 MiB, maxSize=30,0 MiB, persistence=false] [17:05:08,728][INFO][main][IgniteKernal] >>> +----------------------------------------------------------------------+ >>> Ignite ver. 2.7.2#20190116-sha1:fc247e762ac0cd3db350c66516628cd649c64427 >>> +----------------------------------------------------------------------+ >>> OS name: Windows 10 10.0 amd64 >>> CPU(s): 8 >>> Heap: 4.0GB >>> VM name: 14516@work-pc >>> Local node [ID=1CECC333-2890-40BE-A581-AC374128501F, order=1, clientMode=false] >>> Local node addresses: [work-pc.gridgain.local/0:0:0:0:0:0:0:1, /127.0.0.1, /172.25.4.52] >>> Local ports: TCP:10800 TCP:11211 TCP:47100 UDP:47400 TCP:47500 [17:05:08,729][INFO][main][IgniteKernal] >>> Ignite cluster is not active (limited functionality available). Use control.(sh|bat) script or IgniteCluster interface to activate. [17:05:08,729][INFO][main][GridDiscoveryManager] Topology snapshot [ver=1, locNode=1cecc333, servers=1, clients=0, state=INACTIVE, CPUs=8, offheap=3.2GB, heap=4.0GB] [17:06:08,732][INFO][grid-timeout-worker-#23][IgniteKernal] Metrics for local node (to disable set 'metricsLogFrequency' to 0) ^-- Node [id=1cecc333, uptime=00:01:00.006] ^-- H/N/C [hosts=1, nodes=1, CPUs=8] ^-- CPU [cur=0,2%, avg=0,46%, GC=0%] ^-- PageMemory [pages=59] ^-- Heap [used=958MB, free=75,59%, comm=3925MB] ^-- Off-heap [used=0MB, free=99,99%, comm=3487MB] ^-- 30MB_Region_Swapping region [used=0MB, free=100%, comm=15MB] ^-- sysMemPlc region [used=0MB, free=99,98%, comm=100MB] ^-- metastoreMemPlc region [used=0MB, free=99,98%, comm=0MB] ^-- TxLog region [used=0MB, free=100%, comm=100MB] ^-- 40MB_Region_Eviction region [used=0MB, free=100%, comm=20MB] ^-- Default_Region region [used=0MB, free=99,99%, comm=3252MB] ^-- Ignite persistence [used=0MB] ^-- sysMemPlc region [used=0MB] ^-- metastoreMemPlc region [used=0MB] ^-- TxLog region [used=0MB] ^-- Default_Region region [used=0MB] ^-- Outbound messages queue [size=0] ^-- Public thread pool [active=0, idle=0, qSize=0] ^-- System thread pool [active=0, idle=5, qSize=0] [17:07:08,733][INFO][grid-timeout-worker-#23][IgniteKernal] Metrics for local node (to disable set 'metricsLogFrequency' to 0) ^-- Node [id=1cecc333, uptime=00:02:00.013] ^-- H/N/C [hosts=1, nodes=1, CPUs=8] ^-- CPU [cur=0%, avg=0,32%, GC=0%] ^-- PageMemory [pages=59] ^-- Heap [used=967MB, free=75,36%, comm=3925MB] ^-- Off-heap [used=0MB, free=99,99%, comm=3487MB] ^-- 30MB_Region_Swapping region [used=0MB, free=100%, comm=15MB] ^-- sysMemPlc region [used=0MB, free=99,98%, comm=100MB] ^-- metastoreMemPlc region [used=0MB, free=99,98%, comm=0MB] ^-- TxLog region [used=0MB, free=100%, comm=100MB] ^-- 40MB_Region_Eviction region [used=0MB, free=100%, comm=20MB] ^-- Default_Region region [used=0MB, free=99,99%, comm=3252MB] ^-- Ignite persistence [used=0MB] ^-- sysMemPlc region [used=0MB] ^-- metastoreMemPlc region [used=0MB] ^-- TxLog region [used=0MB] ^-- Default_Region region [used=0MB] ^-- Outbound messages queue [size=0] ^-- Public thread pool [active=0, idle=0, qSize=0] ^-- System thread pool [active=0, idle=6, qSize=0] [17:08:08,738][INFO][grid-timeout-worker-#23][IgniteKernal] Metrics for local node (to disable set 'metricsLogFrequency' to 0) ^-- Node [id=1cecc333, uptime=00:03:00.018] ^-- H/N/C [hosts=1, nodes=1, CPUs=8] ^-- CPU [cur=0,13%, avg=0,24%, GC=0%] ^-- PageMemory [pages=59] ^-- Heap [used=976MB, free=75,13%, comm=3925MB] ^-- Off-heap [used=0MB, free=99,99%, comm=3487MB] ^-- 30MB_Region_Swapping region [used=0MB, free=100%, comm=15MB] ^-- sysMemPlc region [used=0MB, free=99,98%, comm=100MB] ^-- metastoreMemPlc region [used=0MB, free=99,98%, comm=0MB] ^-- TxLog region [used=0MB, free=100%, comm=100MB] ^-- 40MB_Region_Eviction region [used=0MB, free=100%, comm=20MB] ^-- Default_Region region [used=0MB, free=99,99%, comm=3252MB] ^-- Ignite persistence [used=0MB] ^-- sysMemPlc region [used=0MB] ^-- metastoreMemPlc region [used=0MB] ^-- TxLog region [used=0MB] ^-- Default_Region region [used=0MB] ^-- Outbound messages queue [size=0] ^-- Public thread pool [active=0, idle=0, qSize=0] ^-- System thread pool [active=0, idle=6, qSize=0]