[2023-07-06T13:08:36,008][WARN ][main][] Property autoActivation deprecated. Use clusterStateOnStart instead. [2023-07-06T13:08:36,140][INFO ][main][IgniteKernal] >>> __________ ________________ >>> / _/ ___/ |/ / _/_ __/ __/ >>> _/ // (7 7 // / / / / _/ >>> /___/\___/_/|_/___/ /_/ /___/ >>> >>> ver. 2.16.0-SNAPSHOT#20230619-sha1:3d3862a7 >>> 2023 Copyright(C) Apache Software Foundation >>> >>> Ignite documentation: https://ignite.apache.org [2023-07-06T13:08:36,145][INFO ][main][IgniteKernal] Config URL: file:/test/nao/server/config/default-config.xml [2023-07-06T13:08:36,166][INFO ][main][IgniteKernal] IgniteConfiguration [igniteInstanceName=null, pubPoolSize=16, svcPoolSize=16, callbackPoolSize=16, stripedPoolSize=16, sysPoolSize=16, mgmtPoolSize=4, dataStreamerPoolSize=16, utilityCachePoolSize=16, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=16, buildIdxPoolSize=4, igniteHome=/test/nao/server, igniteWorkDir=/test/nao/server/work, mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@c667f46, nodeId=650ff665-1971-46e1-93a3-3561e045d6d7, marsh=BinaryMarshaller [], marshLocJobs=false, p2pEnabled=true, netTimeout=5000, netCompressionLevel=1, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, addressFilter=null, sockTimeout=0, ackTimeout=0, marsh=null, reconCnt=10, reconDelay=2000, maxAckTimeout=600000, soLinger=0, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null, skipAddrsRandomization=false], segPlc=USE_FAILURE_HANDLER, segResolveAttempts=2, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=10000, commSpi=TcpCommunicationSpi [connectGate=org.apache.ignite.spi.communication.tcp.internal.ConnectGateway@7159139f, ctxInitLatch=java.util.concurrent.CountDownLatch@65bcf7c2[Count = 1], stopping=false, clientPool=null, nioSrvWrapper=null, stateProvider=null], evtSpi=org.apache.ignite.spi.eventstorage.NoopEventStorageSpi@3db64bd4, colSpi=NoopCollisionSpi [], deploySpi=LocalDeploymentSpi [], indexingSpi=org.apache.ignite.spi.indexing.noop.NoopIndexingSpi@7e75bf2d, addrRslvr=null, encryptionSpi=org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi@385ef531, tracingSpi=org.apache.ignite.spi.tracing.NoopTracingSpi@7fc645e4, clientMode=false, rebalanceThreadPoolSize=4, rebalanceTimeout=10000, rebalanceBatchesPrefetchCnt=3, rebalanceThrottle=0, rebalanceBatchSize=524288, txCfg=TransactionConfiguration [txSerEnabled=false, dfltIsolation=READ_COMMITTED, dfltConcurrency=PESSIMISTIC, dfltTxTimeout=300000, txTimeoutOnPartitionMapExchange=10000, deadlockTimeout=10000, pessimisticTxLogSize=0, pessimisticTxLogLinger=10000, tmLookupClsName=null, txManagerFactory=null, useJtaSync=false], cacheSanityCheckEnabled=true, discoStartupDelay=60000, deployMode=SHARED, p2pMissedCacheSize=100, locHost=null, timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=30000, sysWorkerBlockedTimeout=null, clientFailureDetectionTimeout=30000, metricsLogFreq=0, connectorCfg=ConnectorConfiguration [jettyPath=null, host=0.0.0.0, port=11211, noDelay=true, directBuf=false, sndBufSize=32768, rcvBufSize=32768, idleQryCurTimeout=600000, idleQryCurCheckFreq=60000, sndQueueLimit=0, selectorCnt=4, idleTimeout=180000, sslEnabled=false, sslClientAuth=false, sslFactory=null, portRange=100, threadPoolSize=16, msgInterceptor=null], odbcCfg=null, warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000, cacheMode=PARTITIONED, backups=1, aff=null, grpName=null], classLdr=null, sslCtxFactory=null, platformCfg=null, binaryCfg=null, memCfg=null, pstCfg=null, dsCfg=DataStorageConfiguration [pageSize=1024, concLvl=0, sysDataRegConf=org.apache.ignite.configuration.SystemDataRegionConfiguration@22bd2039, dfltDataRegConf=DataRegionConfiguration [name=default, maxSize=107374182400, initSize=268435456, swapPath=null, pageEvictionMode=DISABLED, pageReplacementMode=CLOCK, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=true, checkpointPageBufSize=0, lazyMemoryAllocation=true, warmUpCfg=null, memoryAllocator=null, cdcEnabled=false], dataRegions=null, storagePath=/test/nao/server/work/data, checkpointFreq=60000, lockWaitTime=10000, checkpointThreads=2, checkpointWriteOrder=SEQUENTIAL, walHistSize=20, maxWalArchiveSize=10737418240, walSegments=10, walSegmentSize=67108864, walPath=/test/nao/server/work/wal, walArchivePath=/test/nao/server/work/wal_archive, cdcWalPath=db/wal/cdc, cdcWalDirMaxSize=0, metricsEnabled=false, walMode=LOG_ONLY, walTlbSize=131072, walBuffSize=0, walFlushFreq=2000, walFsyncDelay=1000, walRecordIterBuffSize=67108864, alwaysWriteFullPages=false, fileIOFactory=org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory@44cb460e, metricsSubIntervalCnt=5, metricsRateTimeInterval=60000, walAutoArchiveAfterInactivity=-1, walForceArchiveTimeout=-1, writeThrottlingEnabled=false, walCompactionEnabled=false, walCompactionLevel=1, checkpointReadLockTimeout=null, walPageCompression=DISABLED, walPageCompressionLevel=null, dfltWarmUpCfg=null, encCfg=org.apache.ignite.configuration.EncryptionConfiguration@2e3cdec2, defragmentationThreadPoolSize=12, minWalArchiveSize=-1, memoryAllocator=null], snapshotPath=/test/nao/backup/snapshot, snapshotThreadPoolSize=10, activeOnStart=true, activeOnStartPropSetFlag=false, autoActivation=false, autoActivationPropSetFlag=true, clusterStateOnStart=null, sqlConnCfg=null, cliConnCfg=ClientConnectorConfiguration [host=0.0.0.0, port=10800, portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true, maxOpenCursorsPerConn=128, threadPoolSize=16, selectorCnt=8, idleTimeout=0, handshakeTimeout=10000, jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false, useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null, thinCliCfg=ThinClientConfiguration [maxActiveTxPerConn=100, maxActiveComputeTasksPerConn=100, sendServerExcStackTraceToClient=false]], mvccVacuumThreadCnt=2, mvccVacuumFreq=5000, authEnabled=false, failureHnd=null, commFailureRslvr=null, sqlCfg=SqlConfiguration [longQryWarnTimeout=3000, dfltQryTimeout=0, sqlQryHistSize=1000, validationEnabled=false], asyncContinuationExecutor=null] [2023-07-06T13:08:36,167][INFO ][main][IgniteKernal] OS: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 [2023-07-06T13:08:36,168][INFO ][main][IgniteKernal] OS user: admin [2023-07-06T13:08:36,171][INFO ][main][IgniteKernal] PID: 11886 [2023-07-06T13:08:36,171][INFO ][main][IgniteKernal] Language runtime: Java Platform API Specification ver. 11 [2023-07-06T13:08:36,172][INFO ][main][IgniteKernal] VM information: OpenJDK Runtime Environment 11.0.18+10-LTS Amazon.com Inc. OpenJDK 64-Bit Server VM 11.0.18+10-LTS [2023-07-06T13:08:36,174][INFO ][main][IgniteKernal] VM total memory: 6.0GB [2023-07-06T13:08:36,174][INFO ][main][IgniteKernal] Remote Management [restart: on, REST: on, JMX (remote: off)] [2023-07-06T13:08:36,175][INFO ][main][IgniteKernal] Logger: Log4J2Logger [quiet=false, config=/test/nao/server/config/ignite-log4j2.xml] [2023-07-06T13:08:36,177][INFO ][main][IgniteKernal] System cache's DataRegion size is configured to 40 MB. Use DataStorageConfiguration.systemRegionInitialSize property to change the setting. [2023-07-06T13:08:36,178][INFO ][main][IgniteKernal] Configured caches [in 'sysMemPlc' dataRegion: ['ignite-sys-cache']] [2023-07-06T13:08:36,178][WARN ][main][IgniteKernal] Peer class loading is enabled (disable it in production for performance and deployment consistency reasons) [2023-07-06T13:08:36,182][INFO ][main][IgniteKernal] 3-rd party licenses can be found at: /test/nao/server/libs/licenses [2023-07-06T13:08:36,254][INFO ][main][IgnitePluginProcessor] Configured plugins: [2023-07-06T13:08:36,254][INFO ][main][IgnitePluginProcessor] ^-- None [2023-07-06T13:08:36,254][INFO ][main][IgnitePluginProcessor] [2023-07-06T13:08:36,259][INFO ][main][FailureProcessor] Configured failure handler: [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]]] [2023-07-06T13:08:36,553][INFO ][main][TcpCommunicationSpi] Successfully bound communication NIO server to TCP port [port=47900, locHost=/10.40.6.208, selectorsCnt=8, selectorSpins=0, pairedConn=false] [2023-07-06T13:08:36,554][WARN ][main][TcpCommunicationSpi] Message queue limit is set to 0 which may lead to potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and receiver sides. [2023-07-06T13:08:36,583][INFO ][main][GridCollisionManager] Collision resolution is disabled (all jobs will be activated upon arrival). [2023-07-06T13:08:36,631][INFO ][main][MaintenanceProcessor] Resolved store directory for node persistent data: /test/nao/server/work/data/ise_devmon_8 [2023-07-06T13:08:36,636][INFO ][main][IgniteKernal] Node is being started in maintenance mode. Starting IsolatedDiscoverySpi instead of configured discovery SPI. [2023-07-06T13:08:36,637][INFO ][main][IgniteKernal] Overriding 'clusterStateOnStart' configuration to 'INACTIVE'. [2023-07-06T13:08:36,662][INFO ][main][CacheObjectBinaryProcessorImpl] Resolved directory for serialized binary metadata: /test/nao/server/work/db/binary_meta/ise_devmon_8 [2023-07-06T13:08:36,699][WARN ][main][GridClusterStateProcessor] Property `autoActivation` will be ignored due to the property `clusterStateOnStart` is presented. [2023-07-06T13:08:36,840][INFO ][main][GridLocalConfigManager] Resolved page store work directory: /test/nao/server/work/data/ise_devmon_8 [2023-07-06T13:08:36,867][INFO ][main][FilePageStoreManager] Resolved page store work directory: /test/nao/server/work/data/ise_devmon_8 [2023-07-06T13:08:36,884][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log work directory: /test/nao/server/work/wal/ise_devmon_8 [2023-07-06T13:08:36,884][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log archive directory: /test/nao/server/work/wal_archive/ise_devmon_8 [2023-07-06T13:08:36,922][INFO ][main][FileHandleManagerImpl] Initialized write-ahead log manager [mode=LOG_ONLY] [2023-07-06T13:08:36,931][INFO ][main][GridCacheDatabaseSharedManager] Region size was reassigned by defragmentation reason: region = 'default', oldInitialSize = '268435456', newInitialSize = '268435456', oldMaxSize = '107374182400', newMaxSize = '42949672960 [2023-07-06T13:08:36,960][INFO ][main][GridCacheDatabaseSharedManager] Configured data regions initialized successfully [total=7] [2023-07-06T13:08:36,989][INFO ][main][IgniteSnapshotManager] Resolved snapshot work directory: /test/nao/backup/snapshot [2023-07-06T13:08:36,989][INFO ][main][IgniteSnapshotManager] Resolved temp directory for snapshot creation: /test/nao/server/work/data/ise_devmon_8/snp [2023-07-06T13:08:37,065][WARN ][main][IgniteH2Indexing] Serialization of Java objects in H2 was enabled. [2023-07-06T13:08:37,253][INFO ][main][ClientListenerProcessor] Client connector processor has started on TCP port 10800 [2023-07-06T13:08:37,320][INFO ][main][GridTcpRestProtocol] Command protocol successfully started [name=TCP binary, host=/0.0.0.0, port=11211] [2023-07-06T13:08:37,413][INFO ][main][IgniteKernal] Non-loopback local IPs: 10.40.6.208, 172.17.0.1 [2023-07-06T13:08:37,414][INFO ][main][IgniteKernal] Enabled local MACs: 0050560AF634, 024206B4ECF3 [2023-07-06T13:08:37,433][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688637187911-146fa746-d583-4c2d-9e75-0a04c8ce2cbe-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688637187911-146fa746-d583-4c2d-9e75-0a04c8ce2cbe-END.bin] [2023-07-06T13:08:37,450][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-06T13:08:37,454][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=3345, fileOff=32861531, len=324899], lastMarked=WALPointer [idx=3345, fileOff=32861531, len=324899], lastCheckpointId=146fa746-d583-4c2d-9e75-0a04c8ce2cbe] [2023-07-06T13:08:37,506][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3345, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@2c06b113, actualFilePtr=WALPointer [idx=3345, fileOff=33186430, len=0]] [2023-07-06T13:08:37,541][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3345, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@234cd86c, actualFilePtr=WALPointer [idx=3345, fileOff=33186430, len=0]] [2023-07-06T13:08:37,542][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=146fa746-d583-4c2d-9e75-0a04c8ce2cbe, pos=WALPointer [idx=3345, fileOff=32861531, len=324899]] [2023-07-06T13:08:37,594][INFO ][main][GridCacheDatabaseSharedManager] Applying lost metastore updates since last checkpoint record [lastMarked=WALPointer [idx=3345, fileOff=32861531, len=324899], lastCheckpointId=146fa746-d583-4c2d-9e75-0a04c8ce2cbe] [2023-07-06T13:08:37,618][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3345, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@9408857, actualFilePtr=WALPointer [idx=3345, fileOff=33186430, len=0]] [2023-07-06T13:08:37,628][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=30 ms] [2023-07-06T13:08:37,628][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-06T13:08:37,638][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=0, partitionsProcessed=0, time=0ms] [2023-07-06T13:08:37,691][INFO ][main][GridEncryptionManager] Encryption keys loaded from metastore. [grps=, masterKeyName=null] [2023-07-06T13:08:37,715][INFO ][main][GridClusterStateProcessor] Restoring history for BaselineTopology[id=0] [2023-07-06T13:08:37,747][INFO ][main][ClusterProcessor] Cluster ID and tag has been read from metastorage: ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-06T13:08:37,751][INFO ][main][IgniteClusterImpl] Shutdown policy was updated [oldVal=null, newVal=null] [2023-07-06T13:08:37,753][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustEnabled' was changed from 'null' to 'false' [2023-07-06T13:08:37,754][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustTimeout' was changed from 'null' to '300000' [2023-07-06T13:08:37,754][INFO ][main][IgniteTxManager] Transactions parameter 'txOwnerDumpRequestsAllowed' was changed from 'null' to 'true' [2023-07-06T13:08:37,754][INFO ][main][IgniteTxManager] Transactions parameter 'longOperationsDumpTimeout' was changed from 'null' to '60000' [2023-07-06T13:08:37,755][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpThreshold' was changed from 'null' to '0' [2023-07-06T13:08:37,755][INFO ][main][IgniteTxManager] Transactions parameter 'transactionTimeDumpSamplesCoefficient' was changed from 'null' to '0.0' [2023-07-06T13:08:37,756][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpSamplesPerSecondLimit' was changed from 'null' to '5' [2023-07-06T13:08:37,756][INFO ][main][IgniteTxManager] Transactions parameter 'collisionsDumpInterval' was changed from 'null' to '1000' [2023-07-06T13:08:37,757][INFO ][main][GridCacheDatabaseSharedManager] Historical rebalance WAL threshold changed [property=historical.rebalance.threshold, oldVal=null, newVal=500] [2023-07-06T13:08:37,758][INFO ][main][GridCacheDatabaseSharedManager] Checkpoint frequency deviation changed [oldVal=null, newVal=null] [2023-07-06T13:08:37,758][INFO ][main][IgniteSnapshotManager] The snapshot transfer rate is not limited. [2023-07-06T13:08:37,759][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.disabledFunctions' was changed from 'null' to '[FILE_WRITE, CANCEL_SESSION, MEMORY_USED, CSVREAD, LINK_SCHEMA, MEMORY_FREE, FILE_READ, CSVWRITE, SESSION_ID, LOCK_MODE]' [2023-07-06T13:08:37,759][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.defaultQueryTimeout' was changed from 'null' to '0' [2023-07-06T13:08:37,760][INFO ][main][IgniteStatisticsManagerImpl] Statistics usage state was changed from null to null [2023-07-06T13:08:37,780][INFO ][main][FilePageStoreManager] Cleanup cache stores [total=1, left=0, cleanFiles=false] [2023-07-06T13:08:40,007][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=59.0 GiB, pages=54679344, tableSize=4.4 GiB, replacementSize=6.5 MiB, checkpointBuffer=2.0 GiB] [2023-07-06T13:08:40,011][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-06T13:08:40,014][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-06T13:08:40,018][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-06T13:08:40,056][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=1.0 GiB, pages=926768, tableSize=76.4 MiB, replacementSize=113.3 KiB, checkpointBuffer=256.0 MiB] [2023-07-06T13:08:40,058][INFO ][main][GridCacheDatabaseSharedManager] Data Regions Started: 7 [2023-07-06T13:08:40,059][INFO ][main][IgniteKernal] ^-- defragPartitionsDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=60416MB, maxCfg=60416MB, usedRam=0MB, freeRam=100%, allocRam=60416MB, allocTotal=0MB] ^-- sysMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- default region [type=default, persistence=true, lazyAlloc=true, ... initCfg=256MB, maxCfg=40960MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB] ^-- metastoreMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- TxLog region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- defragMappingDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=1024MB, maxCfg=1024MB, usedRam=0MB, freeRam=100%, allocRam=1024MB, allocTotal=0MB] ^-- volatileDsMemPlc region [type=user, persistence=false, lazyAlloc=true, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=0MB] [2023-07-06T13:08:40,060][INFO ][main][GridCacheDatabaseSharedManager] Starting binary memory restore for: [-2100569601, -1368047377, -1368047378] [2023-07-06T13:08:41,066][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688637187911-146fa746-d583-4c2d-9e75-0a04c8ce2cbe-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688637187911-146fa746-d583-4c2d-9e75-0a04c8ce2cbe-END.bin] [2023-07-06T13:08:41,066][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=3345, fileOff=32861531, len=324899], lastMarked=WALPointer [idx=3345, fileOff=32861531, len=324899], lastCheckpointId=146fa746-d583-4c2d-9e75-0a04c8ce2cbe] [2023-07-06T13:08:41,096][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3345, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@7772d266, actualFilePtr=WALPointer [idx=3345, fileOff=33186430, len=0]] [2023-07-06T13:08:41,127][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3345, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@b8a144e, actualFilePtr=WALPointer [idx=3345, fileOff=33186430, len=0]] [2023-07-06T13:08:41,127][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=146fa746-d583-4c2d-9e75-0a04c8ce2cbe, pos=WALPointer [idx=3345, fileOff=32861531, len=324899]] [2023-07-06T13:08:41,134][INFO ][main][GridCacheDatabaseSharedManager] Binary memory state restored at node startup [restoredPtr=WALPointer [idx=3345, fileOff=33186430, len=0]] [2023-07-06T13:08:41,139][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/nao/server/work/wal/ise_devmon_8/0000000000000005.wal, offset=33186430, ver=2] [2023-07-06T13:08:41,228][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=ignite-sys-cache, id=-2100569601, dataRegionName=sysMemPlc, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647, mvcc=false] [2023-07-06T13:08:42,737][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=40.0 GiB, pages=37070736, tableSize=3.0 GiB, replacementSize=4.4 MiB, checkpointBuffer=2.0 GiB] [2023-07-06T13:08:42,752][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=cache1, id=-1368047377, dataRegionName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647, mvcc=false] [2023-07-06T13:08:42,755][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=cache0, id=-1368047378, dataRegionName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647, mvcc=false] [2023-07-06T13:08:42,757][INFO ][main][GridCacheDatabaseSharedManager] Binary recovery performed in 2697 ms. [2023-07-06T13:08:42,759][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688637187911-146fa746-d583-4c2d-9e75-0a04c8ce2cbe-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688637187911-146fa746-d583-4c2d-9e75-0a04c8ce2cbe-END.bin] [2023-07-06T13:08:42,760][INFO ][main][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=WALPointer [idx=3345, fileOff=32861531, len=324899], lastCheckpointId=146fa746-d583-4c2d-9e75-0a04c8ce2cbe] [2023-07-06T13:08:42,785][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3345, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@2c2e3460, actualFilePtr=WALPointer [idx=3345, fileOff=33186459, len=0]] [2023-07-06T13:08:42,795][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=40 ms] [2023-07-06T13:08:42,796][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-06T13:08:51,328][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=3, partitionsProcessed=17096, time=8s] [2023-07-06T13:08:51,332][INFO ][main][MaintenanceProcessor] Node requires maintenance, non-empty set of maintenance tasks is found: [defragmentationMaintenanceTask] [2023-07-06T13:08:51,335][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/nao/server/work/wal/ise_devmon_8/0000000000000005.wal, offset=33186459, ver=2] [2023-07-06T13:08:51,678][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint started [checkpointId=344bac2c-a70f-4e99-95f4-e4e59f58aa7a, startPtr=WALPointer [idx=3345, fileOff=51151231, len=324899], checkpointBeforeLockTime=181ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=51ms, checkpointLockHoldTime=85ms, walCpRecordFsyncDuration=37ms, writeCheckpointEntryDuration=15ms, splitAndSortCpPagesDuration=21ms, pages=16996, reason='node started'] [2023-07-06T13:08:56,562][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint finished [cpId=344bac2c-a70f-4e99-95f4-e4e59f58aa7a, pages=16996, markPos=WALPointer [idx=3345, fileOff=51151231, len=324899], walSegmentsCovered=[], markDuration=161ms, pagesWrite=351ms, fsync=4531ms, total=5224ms] [2023-07-06T13:08:56,676][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=53ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=34ms, checkpointLockHoldTime=59ms, reason='beforeDefragmentation'] [2023-07-06T13:08:56,690][INFO ][disco-notifier-worker-#98][MvccProcessorImpl] Assigned mvcc coordinator [crd=MvccCoordinator [topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0], nodeId=650ff665-1971-46e1-93a3-3561e045d6d7, ver=1688638116639, local=true, initialized=false]] [2023-07-06T13:08:56,695][INFO ][defragmentation-thread][CachePartitionDefragmentationManager] Defragmentation started. [2023-07-06T13:08:56,695][INFO ][sys-#77][ClusterProcessor] Writing cluster ID and tag to metastorage on ready for write ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-06T13:08:56,700][INFO ][defragmentation-thread][CacheGroupContext] Local state for group durability has changed [name=cache0, enabled=false] [2023-07-06T13:08:56,715][INFO ][defragmentation-worker-#109][CachePartitionDefragmentationManager] Skipping already defragmented partition [grpId=-1368047378, partId=9, partFileName=part-dfrg-9.bin, mappingFileName=part-map-9.bin, workDir=/test/nao/server/work/data/ise_devmon_8/cache-cache0] [2023-07-06T13:08:56,910][INFO ][main][IgniteKernal] Security status [authentication=off, sandbox=off, tls/ssl=off] [2023-07-06T13:08:56,911][INFO ][main][IgniteKernal] Performance suggestions for grid (fix if possible) [2023-07-06T13:08:56,911][INFO ][main][IgniteKernal] To disable, set -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true [2023-07-06T13:08:56,911][INFO ][main][IgniteKernal] ^-- Set max direct memory size if getting 'OOME: Direct buffer memory' (add '-XX:MaxDirectMemorySize=[g|G|m|M|k|K]' to JVM options) [2023-07-06T13:08:56,912][INFO ][main][IgniteKernal] ^-- Speed up flushing of dirty pages by OS (alter vm.dirty_expire_centisecs parameter by setting to 500) [2023-07-06T13:08:56,912][INFO ][main][IgniteKernal] ^-- Reduce pages swapping ratio (set vm.swappiness=10 or less) [2023-07-06T13:08:56,912][INFO ][main][IgniteKernal] ^-- Disable assertions (remove '-ea' from JVM options) [2023-07-06T13:08:56,912][INFO ][main][IgniteKernal] Refer to this page for more performance suggestions: https://ignite.apache.org/docs/latest/perf-and-troubleshooting/memory-tuning [2023-07-06T13:08:56,912][INFO ][main][IgniteKernal] [2023-07-06T13:08:56,912][INFO ][main][IgniteKernal] Classpath value: /test/nao/server/libs/log4j-core-2.20.0.jar:/test/nao/server/libs/log4j-api-2.20.0.jar:/test/nao/server/libs/ignite-log4j2-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-core-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/annotations-16.0.3.jar:/test/nao/server/libs/cache-api-1.0.0.jar:/test/nao/server/libs/ignite-control-utility/ignite-control-utility-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-indexing/h2-1.4.197.jar:/test/nao/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar:/test/nao/server/libs/ignite-indexing/ignite-indexing-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-indexing/lucene-core-8.11.2.jar:/test/nao/server/libs/ignite-indexing/commons-codec-1.15.jar:/test/nao/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar:/test/nao/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/ignite-spring-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/commons-logging-1.1.1.jar:/test/nao/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar:/test/nao/server/libs/licenses/* [2023-07-06T13:08:56,913][INFO ][main][IgniteKernal] List of files containing in classpath: /test/nao/server/libs/log4j-core-2.20.0.jar;/test/nao/server/libs/log4j-api-2.20.0.jar;/test/nao/server/libs/ignite-log4j2-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-core-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/annotations-16.0.3.jar;/test/nao/server/libs/cache-api-1.0.0.jar;/test/nao/server/libs/ignite-control-utility/ignite-control-utility-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-indexing/h2-1.4.197.jar;/test/nao/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar;/test/nao/server/libs/ignite-indexing/ignite-indexing-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-indexing/lucene-core-8.11.2.jar;/test/nao/server/libs/ignite-indexing/commons-codec-1.15.jar;/test/nao/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar;/test/nao/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/ignite-spring-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/commons-logging-1.1.1.jar;/test/nao/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar; [2023-07-06T13:08:56,980][INFO ][main][IgniteKernal] >>> +--------------------------------------------------------------------------------+ >>> Ignite ver. 2.16.0-SNAPSHOT#20230619-sha1:3d3862a7272476fd008fd5f2d9eee2a33db164eb >>> +--------------------------------------------------------------------------------+ >>> OS name: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 >>> CPU(s): -1 >>> Heap: 0.1GB >>> VM name: 11886@devmon-8 >>> Local node [ID=650FF665-1971-46E1-93A3-3561E045D6D7, order=1, clientMode=false] >>> Local node addresses: [localhost/127.0.0.1] >>> Local ports: TCP:10800 TCP:11211 TCP:47900 >>> +--------------------------------------------------------------------------------+ [2023-07-06T13:08:56,980][INFO ][main][IgniteKernal] >>> Ignite cluster is in INACTIVE state (limited functionality available). Use control.(sh|bat) script or IgniteCluster.state(ClusterState.ACTIVE) to change the state. [2023-07-06T13:08:56,988][INFO ][main][GridDiscoveryManager] Topology snapshot [ver=1, locNode=650ff665, servers=1, clients=0, state=INACTIVE, CPUs=-1, offheap=40.0GB, heap=0.1GB, aliveNodes=[IsolatedNode [id=650ff665-1971-46e1-93a3-3561e045d6d7, consistentId=ise_devmon-8, isClient=false, ver=2.16.0#20230619-sha1:3d3862a7]]] [2023-07-06T13:08:56,991][INFO ][main][GridDiscoveryManager] ^-- Baseline [id=0, size=1, online=1, offline=0] [2023-07-06T13:08:56,992][INFO ][main][G] Node started : [stage="Configure system pool" (21 ms),stage="Start managers" (544 ms),stage="Configure binary metadata" (37 ms),stage="Start processors" (715 ms),stage="Init metastore" (372 ms),stage="Init and start regions" (2278 ms),stage="Restore binary memory" (2698 ms),stage="Restore logical state" (8571 ms),stage="Finish recovery" (2 ms),stage="Join topology" (5357 ms),stage="Await transition" (12 ms),stage="Await exchange" (290 ms),stage="Total time" (20897 ms)] [2023-07-06T13:09:17,035][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=78ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=1837ms, pages=3395825, reason='partition defragmented'] [2023-07-06T13:10:07,178][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=22ms, reason='timeout'] [2023-07-06T13:10:37,593][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=3395825, markPos=, walSegmentsCovered=, markDuration=1839ms, pagesWrite=79727ms, fsync=829ms, total=82473ms] [2023-07-06T13:10:43,490][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=289ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=5608ms, pages=15129408, reason='partition defragmented'] [2023-07-06T13:10:57,512][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=26ms, reason='timeout'] [2023-07-06T13:11:13,434][WARN ][defragmentation-worker-#105][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragMappingDataRegion [2023-07-06T13:11:59,052][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=17ms, reason='timeout'] [2023-07-06T13:12:28,481][WARN ][defragmentation-worker-#100][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): default [2023-07-06T13:12:48,140][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=13ms, reason='timeout'] [2023-07-06T13:13:51,011][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=11ms, reason='timeout'] [2023-07-06T13:14:58,057][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=12ms, reason='timeout'] [2023-07-06T13:15:33,356][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=15129408, markPos=, walSegmentsCovered=, markDuration=5608ms, pagesWrite=288747ms, fsync=1119ms, total=295763ms] [2023-07-06T13:15:33,361][WARN ][defragmentation-worker-#109][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:33,395][WARN ][defragmentation-worker-#109][FailureProcessor] No deadlocked threads detected. [2023-07-06T13:15:35,783][WARN ][jvm-pause-detector-worker][IgniteKernal] Possible too long JVM pause: 2360 milliseconds. [2023-07-06T13:15:35,832][WARN ][defragmentation-worker-#109][FailureProcessor] Thread dump at 2023/07/06 13:15:35 MSK Thread [name="main", id=1, state=WAITING, blockCnt=8, waitCnt=17272] Lock [object=java.util.concurrent.CountDownLatch$Sync@164473cb, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345) at java.base@11.0.18/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232) at app//o.a.i.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:398) Locked synchronizers: java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@36b8db00 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@57725775 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@4fb8b8c5 Thread [name="Reference Handler", id=2, state=RUNNABLE, blockCnt=3, waitCnt=0] at java.base@11.0.18/java.lang.ref.Reference.waitForReferencePendingList(Native Method) at java.base@11.0.18/java.lang.ref.Reference.processPendingReferences(Reference.java:241) at java.base@11.0.18/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:213) Thread [name="Finalizer", id=3, state=WAITING, blockCnt=4, waitCnt=5] Lock [object=java.lang.ref.ReferenceQueue$Lock@6654dad8, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at java.base@11.0.18/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:170) Thread [name="Signal Dispatcher", id=4, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Common-Cleaner", id=19, state=TIMED_WAITING, blockCnt=13, waitCnt=17] Lock [object=java.lang.ref.ReferenceQueue$Lock@2358f098, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.18/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) at java.base@11.0.18/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:161) Thread [name="Log4j2-TF-6-Scheduled-2", id=23, state=TIMED_WAITING, blockCnt=0, waitCnt=7] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@29ac1aa6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="jvm-pause-detector-worker", id=24, state=TIMED_WAITING, blockCnt=0, waitCnt=8213] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:124) Thread [name="ignite-clock", id=25, state=TIMED_WAITING, blockCnt=0, waitCnt=40545] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils$25.run(IgniteUtils.java:3641) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-0-#1", id=26, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-1-#2", id=27, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-2-#3", id=28, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-3-#4", id=29, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-4-#5", id=30, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-5-#6", id=31, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-6-#7", id=32, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-7-#8", id=33, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-8-#9", id=34, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-9-#10", id=35, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-10-#11", id=36, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-11-#12", id=37, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-12-#13", id=38, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-13-#14", id=39, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-14-#15", id=40, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-15-#16", id=41, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-0-#17", id=42, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-1-#18", id=43, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-2-#19", id=44, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-3-#20", id=45, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-4-#21", id=46, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-5-#22", id=47, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-6-#23", id=48, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-7-#24", id=49, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-8-#25", id=50, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-9-#26", id=51, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-10-#27", id=52, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-11-#28", id=53, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-12-#29", id=54, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-13-#30", id=55, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-14-#31", id=56, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-15-#32", id=57, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-timeout-worker-#38", id=63, state=TIMED_WAITING, blockCnt=3, waitCnt=1974] Lock [object=java.lang.Object@1cc3fe73, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:269) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-comm-#47%TcpCommunicationSpi%", id=72, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@5c016529 - locked sun.nio.ch.EPollSelectorImpl@38398201 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-0-#39%TcpCommunicationSpi%", id=64, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7189c68a - locked sun.nio.ch.EPollSelectorImpl@72ba9eae at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-1-#40%TcpCommunicationSpi%", id=65, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@25c595fd - locked sun.nio.ch.EPollSelectorImpl@519832ce at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-2-#41%TcpCommunicationSpi%", id=66, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6e1a13e1 - locked sun.nio.ch.EPollSelectorImpl@48f3bf62 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-3-#42%TcpCommunicationSpi%", id=67, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4190b1fc - locked sun.nio.ch.EPollSelectorImpl@510ee542 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-4-#43%TcpCommunicationSpi%", id=68, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6f357199 - locked sun.nio.ch.EPollSelectorImpl@3a68cbb2 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-5-#44%TcpCommunicationSpi%", id=69, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7e339252 - locked sun.nio.ch.EPollSelectorImpl@58315c23 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-6-#45%TcpCommunicationSpi%", id=70, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1ccc4705 - locked sun.nio.ch.EPollSelectorImpl@60053a75 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-7-#46%TcpCommunicationSpi%", id=71, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2c69525 - locked sun.nio.ch.EPollSelectorImpl@5044011a at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="tcp-comm-worker-#1-#48", id=73, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e4e9078, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at app//o.a.i.spi.communication.tcp.internal.CommunicationWorker.body(CommunicationWorker.java:167) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at app//o.a.i.spi.communication.tcp.TcpCommunicationSpi$3.body(TcpCommunicationSpi.java:848) at app//o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:58) Thread [name="binary-metadata-writer-#49", id=75, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@67d1c156, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body0(BinaryMetadataFileStore.java:477) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body(BinaryMetadataFileStore.java:458) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="page-lock-tracker-timeout", id=76, state=TIMED_WAITING, blockCnt=0, waitCnt=7] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.worker.CycleThread.run(CycleThread.java:47) Thread [name="long-qry-#50", id=77, state=TIMED_WAITING, blockCnt=0, waitCnt=416] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:8345) at app//o.a.i.i.processors.query.running.HeavyQueriesTracker$1.body(HeavyQueriesTracker.java:104) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-rest-#64", id=91, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@51880f34 - locked sun.nio.ch.EPollSelectorImpl@21b488ce at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-0-#60", id=87, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7ba301d0 - locked sun.nio.ch.EPollSelectorImpl@1211a483 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-1-#61", id=88, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@371f85a - locked sun.nio.ch.EPollSelectorImpl@686e202 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-2-#62", id=89, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6548de83 - locked sun.nio.ch.EPollSelectorImpl@7ae24b15 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-3-#63", id=90, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@765ddf3b - locked sun.nio.ch.EPollSelectorImpl@477ab500 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-data-loader-flusher-#68", id=95, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6465d7aa, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.DelayQueue.take(DelayQueue.java:217) at app//o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java:106) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="db-checkpoint-thread-defrag-#69", id=96, state=RUNNABLE, blockCnt=4, waitCnt=9] at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointWorkflow.splitAndSortCpPagesIfNeeded(CheckpointWorkflow.java:489) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointWorkflow.markCheckpointBegin(CheckpointWorkflow.java:358) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.doCheckpoint(Checkpointer.java:420) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:278) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="ttl-cleanup-worker-#73", id=101, state=WAITING, blockCnt=1, waitCnt=2] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:142) at app//o.a.i.i.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:164) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-6", id=122, state=TIMED_WAITING, blockCnt=41, waitCnt=5966133] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-5", id=121, state=TIMED_WAITING, blockCnt=43, waitCnt=5663964] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-9", id=125, state=TIMED_WAITING, blockCnt=34, waitCnt=5588940] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="db-checkpoint-thread-#93", id=137, state=TIMED_WAITING, blockCnt=0, waitCnt=36] Lock [object=o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer@148b207a, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.waitCheckpointEvent(Checkpointer.java:820) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:260) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#94", id=138, state=WAITING, blockCnt=1, waitCnt=519] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64b555e, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#95", id=139, state=WAITING, blockCnt=0, waitCnt=870] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64b555e, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#96", id=140, state=WAITING, blockCnt=3, waitCnt=8578] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4cd91aba, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#97", id=141, state=WAITING, blockCnt=4, waitCnt=8412] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4cd91aba, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-thread", id=142, state=BLOCKED, blockCnt=2, waitCnt=3754026] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11657) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11559) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11539) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction$$Lambda$940/0x0000000800717c40.run(Unknown Source) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="disco-notifier-worker-#98", id=143, state=WAITING, blockCnt=0, waitCnt=4] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@318dd7e0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2830) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2875) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-1", id=144, state=WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="disco-event-worker-#99", id=145, state=WAITING, blockCnt=0, waitCnt=3] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@69b84eec, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:3089) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:3060) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-2", id=146, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-3", id=147, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-worker-#100", id=148, state=BLOCKED, blockCnt=3, waitCnt=3729341] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@578c6d85 Thread [name="defragmentation-worker-#101", id=149, state=BLOCKED, blockCnt=5, waitCnt=3728152] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@3e0c2a43 Thread [name="defragmentation-worker-#102", id=150, state=BLOCKED, blockCnt=5, waitCnt=3741206] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6930c0bf Thread [name="defragmentation-worker-#103", id=151, state=BLOCKED, blockCnt=6, waitCnt=3746745] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@4a151ef5 Thread [name="defragmentation-worker-#104", id=152, state=BLOCKED, blockCnt=4, waitCnt=3746701] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5b23f81a Thread [name="defragmentation-worker-#105", id=153, state=BLOCKED, blockCnt=4, waitCnt=3741971] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@f006ffb Thread [name="defragmentation-worker-#106", id=154, state=BLOCKED, blockCnt=4, waitCnt=3754045] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@30bd395b Thread [name="defragmentation-worker-#107", id=155, state=BLOCKED, blockCnt=5, waitCnt=3755015] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5092ba62 Thread [name="defragmentation-worker-#108", id=156, state=BLOCKED, blockCnt=2, waitCnt=3759891] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@2a626975 Thread [name="defragmentation-worker-#109", id=157, state=RUNNABLE, blockCnt=0, waitCnt=3736223] at java.management@11.0.18/sun.management.ThreadImpl.dumpThreads0(Native Method) at java.management@11.0.18/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:521) at java.management@11.0.18/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:509) at app//o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1529) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:205) - locked o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3 at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@381dcd99 Thread [name="defragmentation-worker-#110", id=158, state=BLOCKED, blockCnt=2, waitCnt=3740853] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@1558a7c0 Thread [name="defragmentation-worker-#111", id=159, state=BLOCKED, blockCnt=2, waitCnt=3766740] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#109, ownerId=157] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@771d2a0b Thread [name="exchange-worker-#112", id=160, state=TIMED_WAITING, blockCnt=0, waitCnt=80] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@504eb410, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) at java.base@11.0.18/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:3253) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:3182) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-client-listener-#113", id=161, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@fa35b2e - locked sun.nio.ch.EPollSelectorImpl@66d30625 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-0-#51", id=78, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4db7eb3d - locked sun.nio.ch.EPollSelectorImpl@687c28af at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-1-#52", id=79, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7d9364a5 - locked sun.nio.ch.EPollSelectorImpl@38fa71c at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-2-#53", id=80, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@71135a53 - locked sun.nio.ch.EPollSelectorImpl@78232d8e at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-3-#54", id=81, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1e73c863 - locked sun.nio.ch.EPollSelectorImpl@603a3bf6 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-4-#55", id=82, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@b902460 - locked sun.nio.ch.EPollSelectorImpl@6a3e27c1 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-5-#56", id=83, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6a8eca35 - locked sun.nio.ch.EPollSelectorImpl@5b5ab919 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-6-#57", id=84, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@51e33536 - locked sun.nio.ch.EPollSelectorImpl@4a7800e4 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-7-#58", id=85, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@563bb4fc - locked sun.nio.ch.EPollSelectorImpl@58a74d53 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="services-deployment-worker-#114", id=162, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@102599f4, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.service.ServiceDeploymentManager$ServicesDeploymentWorker.body(ServiceDeploymentManager.java:466) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="session-timeout-worker-#59", id=86, state=TIMED_WAITING, blockCnt=0, waitCnt=397] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.processors.rest.GridRestProcessor$4.body(GridRestProcessor.java:504) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#115", id=173, state=WAITING, blockCnt=0, waitCnt=9374543] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ad0c434, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#116", id=174, state=WAITING, blockCnt=0, waitCnt=9396556] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ad0c434, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#153", id=223, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="stat-mgmt-#154", id=224, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@73681568, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#155", id=225, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#156", id=226, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#157", id=227, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#158", id=228, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#159", id=229, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#160", id=230, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-22", id=231, state=TIMED_WAITING, blockCnt=1, waitCnt=145084] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#161", id=232, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) [2023-07-06T13:15:35,861][WARN ][defragmentation-worker-#109][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:35,862][ERROR][defragmentation-worker-#109][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,862][WARN ][defragmentation-worker-#103][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,863][INFO ][defragmentation-worker-#103][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:35,877][WARN ][defragmentation-worker-#103][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:35,877][ERROR][defragmentation-worker-#103][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,878][WARN ][defragmentation-worker-#110][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,879][INFO ][defragmentation-worker-#110][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:35,890][WARN ][defragmentation-worker-#110][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:35,890][ERROR][defragmentation-worker-#110][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,890][WARN ][defragmentation-worker-#108][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,891][INFO ][defragmentation-worker-#108][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:35,965][WARN ][defragmentation-worker-#108][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:35,965][WARN ][defragmentation-worker-#100][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,966][INFO ][defragmentation-worker-#100][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:35,965][ERROR][defragmentation-worker-#108][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,979][WARN ][defragmentation-worker-#100][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:35,979][ERROR][defragmentation-worker-#100][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,980][WARN ][defragmentation-worker-#104][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,980][INFO ][defragmentation-worker-#104][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:35,992][WARN ][defragmentation-worker-#104][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:35,992][ERROR][defragmentation-worker-#104][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,993][WARN ][defragmentation-worker-#101][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:35,994][INFO ][defragmentation-worker-#101][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:36,034][WARN ][defragmentation-worker-#101][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#108, id=156], state=RUNNABLE Locked pages = [282011847838040[0001007d00034958](r=0|w=1)] Locked pages log: name=defragmentation-worker-#108 time=(1688638535994, 2023-07-06 13:15:35.994) L=1 -> Write lock pageId=282011847838040, structureId=cache0-125 [pageIdHex=0001007d00034958, partId=125, pageIdx=215384, flags=00000001] [2023-07-06T13:15:36,035][ERROR][defragmentation-worker-#101][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,036][WARN ][defragmentation-worker-#102][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,036][INFO ][defragmentation-worker-#102][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:36,050][WARN ][defragmentation-worker-#102][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:36,051][ERROR][defragmentation-worker-#102][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,052][WARN ][defragmentation-thread][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11657) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11559) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11539) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,052][INFO ][defragmentation-thread][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:36,071][WARN ][defragmentation-thread][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#108, id=156], state=RUNNABLE Locked pages = [1126436777869961[0004007d0001c289](r=1|w=0)] Locked pages log: name=defragmentation-worker-#108 time=(1688638536053, 2023-07-06 13:15:36.053) L=1 -> Read lock pageId=1126436777869961, structureId=cache0-p-125##CacheData [pageIdHex=0004007d0001c289, partId=125, pageIdx=115337, flags=00000004] [2023-07-06T13:15:36,071][ERROR][defragmentation-thread][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11657) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11559) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11539) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,072][WARN ][defragmentation-worker-#105][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,073][INFO ][defragmentation-worker-#105][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:36,088][WARN ][defragmentation-worker-#105][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#103, id=151], state=RUNNABLE Locked pages = [282041912597617[0001008400031c71](r=0|w=1)] Locked pages log: name=defragmentation-worker-#103 time=(1688638536073, 2023-07-06 13:15:36.073) L=1 -> Write lock pageId=282041912597617, structureId=cache0-132 [pageIdHex=0001008400031c71, partId=132, pageIdx=203889, flags=00000001] Thread=[name=defragmentation-worker-#108, id=156], state=RUNNABLE Locked pages = [282011847630659[0001007d00001f43](r=1|w=0)] Locked pages log: name=defragmentation-worker-#108 time=(1688638536073, 2023-07-06 13:15:36.073) L=1 -> Read lock pageId=282011847630659, structureId=link-map [pageIdHex=0001007d00001f43, partId=125, pageIdx=8003, flags=00000001] [2023-07-06T13:15:36,088][ERROR][defragmentation-worker-#105][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,088][WARN ][defragmentation-worker-#107][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,089][INFO ][defragmentation-worker-#107][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:36,103][WARN ][defragmentation-worker-#107][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#104, id=152], state=RUNNABLE Locked pages = [282029027700096[0001008100032d80](r=0|w=1)] Locked pages log: name=defragmentation-worker-#104 time=(1688638536089, 2023-07-06 13:15:36.089) L=1 -> Write lock pageId=282029027700096, structureId=cache0-129 [pageIdHex=0001008100032d80, partId=129, pageIdx=208256, flags=00000001] [2023-07-06T13:15:36,103][ERROR][defragmentation-worker-#107][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,104][WARN ][defragmentation-worker-#111][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,104][INFO ][defragmentation-worker-#111][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:36,118][WARN ][defragmentation-worker-#111][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:15:36,119][ERROR][defragmentation-worker-#111][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,119][WARN ][defragmentation-worker-#106][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:36,120][INFO ][defragmentation-worker-#106][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:15:36,137][WARN ][defragmentation-worker-#106][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#104, id=152], state=RUNNABLE Locked pages = [282029027700362[0001008100032e8a](r=0|w=1)] Locked pages log: name=defragmentation-worker-#104 time=(1688638536120, 2023-07-06 13:15:36.120) L=1 -> Write lock pageId=282029027700362, structureId=cache0-129 [pageIdHex=0001008100032e8a, partId=129, pageIdx=208522, flags=00000001] L=2 -> Write lock pageId=1126453957831234, structureId=cache0-129 [pageIdHex=0004008100032a42, partId=129, pageIdx=207426, flags=00000004] L=1 <- Write unlock pageId=1126453957831234, structureId=cache0-129 [pageIdHex=0004008100032a42, partId=129, pageIdx=207426, flags=00000004] [2023-07-06T13:15:36,138][ERROR][defragmentation-worker-#106][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:15:43,514][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=1ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=10157ms, pages=18502224, reason='partition defragmented'] [2023-07-06T13:16:05,423][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=10ms, reason='timeout'] [2023-07-06T13:17:53,875][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=10ms, reason='timeout'] [2023-07-06T13:18:49,020][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=1ms, checkpointLockHoldTime=17ms, reason='timeout'] [2023-07-06T13:19:45,776][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=18ms, reason='timeout'] [2023-07-06T13:20:36,920][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=11ms, reason='timeout'] [2023-07-06T13:21:16,572][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=18502224, markPos=, walSegmentsCovered=, markDuration=10158ms, pagesWrite=332344ms, fsync=713ms, total=343216ms] [2023-07-06T13:21:16,573][WARN ][defragmentation-worker-#103][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:16,574][WARN ][defragmentation-worker-#103][FailureProcessor] No deadlocked threads detected. [2023-07-06T13:21:18,251][WARN ][jvm-pause-detector-worker][IgniteKernal] Possible too long JVM pause: 1651 milliseconds. [2023-07-06T13:21:18,254][WARN ][defragmentation-worker-#103][FailureProcessor] Thread dump at 2023/07/06 13:21:18 MSK Thread [name="main", id=1, state=WAITING, blockCnt=8, waitCnt=17272] Lock [object=java.util.concurrent.CountDownLatch$Sync@164473cb, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345) at java.base@11.0.18/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232) at app//o.a.i.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:398) Locked synchronizers: java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@36b8db00 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@57725775 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@4fb8b8c5 Thread [name="Reference Handler", id=2, state=RUNNABLE, blockCnt=3, waitCnt=0] at java.base@11.0.18/java.lang.ref.Reference.waitForReferencePendingList(Native Method) at java.base@11.0.18/java.lang.ref.Reference.processPendingReferences(Reference.java:241) at java.base@11.0.18/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:213) Thread [name="Finalizer", id=3, state=WAITING, blockCnt=4, waitCnt=5] Lock [object=java.lang.ref.ReferenceQueue$Lock@6654dad8, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at java.base@11.0.18/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:170) Thread [name="Signal Dispatcher", id=4, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Common-Cleaner", id=19, state=TIMED_WAITING, blockCnt=16, waitCnt=25] Lock [object=java.lang.ref.ReferenceQueue$Lock@2358f098, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.18/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) at java.base@11.0.18/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:161) Thread [name="Log4j2-TF-6-Scheduled-2", id=23, state=TIMED_WAITING, blockCnt=0, waitCnt=13] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@29ac1aa6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="jvm-pause-detector-worker", id=24, state=TIMED_WAITING, blockCnt=0, waitCnt=14941] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:124) Thread [name="ignite-clock", id=25, state=TIMED_WAITING, blockCnt=0, waitCnt=73805] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils$25.run(IgniteUtils.java:3641) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-0-#1", id=26, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-1-#2", id=27, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-2-#3", id=28, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-3-#4", id=29, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-4-#5", id=30, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-5-#6", id=31, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-6-#7", id=32, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-7-#8", id=33, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-8-#9", id=34, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-9-#10", id=35, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-10-#11", id=36, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-11-#12", id=37, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-12-#13", id=38, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-13-#14", id=39, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-14-#15", id=40, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-15-#16", id=41, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-0-#17", id=42, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-1-#18", id=43, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-2-#19", id=44, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-3-#20", id=45, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-4-#21", id=46, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-5-#22", id=47, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-6-#23", id=48, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-7-#24", id=49, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-8-#25", id=50, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-9-#26", id=51, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-10-#27", id=52, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-11-#28", id=53, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-12-#29", id=54, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-13-#30", id=55, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-14-#31", id=56, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-15-#32", id=57, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-timeout-worker-#38", id=63, state=TIMED_WAITING, blockCnt=3, waitCnt=3777] Lock [object=java.lang.Object@1cc3fe73, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:269) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-comm-#47%TcpCommunicationSpi%", id=72, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@5c016529 - locked sun.nio.ch.EPollSelectorImpl@38398201 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-0-#39%TcpCommunicationSpi%", id=64, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7189c68a - locked sun.nio.ch.EPollSelectorImpl@72ba9eae at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-1-#40%TcpCommunicationSpi%", id=65, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@25c595fd - locked sun.nio.ch.EPollSelectorImpl@519832ce at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-2-#41%TcpCommunicationSpi%", id=66, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6e1a13e1 - locked sun.nio.ch.EPollSelectorImpl@48f3bf62 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-3-#42%TcpCommunicationSpi%", id=67, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4190b1fc - locked sun.nio.ch.EPollSelectorImpl@510ee542 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-4-#43%TcpCommunicationSpi%", id=68, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6f357199 - locked sun.nio.ch.EPollSelectorImpl@3a68cbb2 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-5-#44%TcpCommunicationSpi%", id=69, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7e339252 - locked sun.nio.ch.EPollSelectorImpl@58315c23 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-6-#45%TcpCommunicationSpi%", id=70, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1ccc4705 - locked sun.nio.ch.EPollSelectorImpl@60053a75 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-7-#46%TcpCommunicationSpi%", id=71, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2c69525 - locked sun.nio.ch.EPollSelectorImpl@5044011a at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="tcp-comm-worker-#1-#48", id=73, state=TIMED_WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e4e9078, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at app//o.a.i.spi.communication.tcp.internal.CommunicationWorker.body(CommunicationWorker.java:167) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at app//o.a.i.spi.communication.tcp.TcpCommunicationSpi$3.body(TcpCommunicationSpi.java:848) at app//o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:58) Thread [name="binary-metadata-writer-#49", id=75, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@67d1c156, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body0(BinaryMetadataFileStore.java:477) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body(BinaryMetadataFileStore.java:458) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="page-lock-tracker-timeout", id=76, state=TIMED_WAITING, blockCnt=0, waitCnt=13] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.worker.CycleThread.run(CycleThread.java:47) Thread [name="long-qry-#50", id=77, state=TIMED_WAITING, blockCnt=0, waitCnt=757] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:8345) at app//o.a.i.i.processors.query.running.HeavyQueriesTracker$1.body(HeavyQueriesTracker.java:104) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-rest-#64", id=91, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@51880f34 - locked sun.nio.ch.EPollSelectorImpl@21b488ce at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-0-#60", id=87, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7ba301d0 - locked sun.nio.ch.EPollSelectorImpl@1211a483 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-1-#61", id=88, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@371f85a - locked sun.nio.ch.EPollSelectorImpl@686e202 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-2-#62", id=89, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6548de83 - locked sun.nio.ch.EPollSelectorImpl@7ae24b15 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-3-#63", id=90, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@765ddf3b - locked sun.nio.ch.EPollSelectorImpl@477ab500 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-data-loader-flusher-#68", id=95, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6465d7aa, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.DelayQueue.take(DelayQueue.java:217) at app//o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java:106) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="db-checkpoint-thread-defrag-#69", id=96, state=RUNNABLE, blockCnt=6, waitCnt=12] at java.base@11.0.18/java.util.concurrent.ConcurrentHashMap$Traverser.advance(ConcurrentHashMap.java:3374) at java.base@11.0.18/java.util.concurrent.ConcurrentHashMap$KeyIterator.next(ConcurrentHashMap.java:3454) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointWorkflow.splitAndSortCpPagesIfNeeded(CheckpointWorkflow.java:484) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointWorkflow.markCheckpointBegin(CheckpointWorkflow.java:358) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.doCheckpoint(Checkpointer.java:420) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:278) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="ttl-cleanup-worker-#73", id=101, state=WAITING, blockCnt=1, waitCnt=2] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:142) at app//o.a.i.i.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:164) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-5", id=121, state=TIMED_WAITING, blockCnt=72, waitCnt=10571715] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="db-checkpoint-thread-#93", id=137, state=TIMED_WAITING, blockCnt=0, waitCnt=60] Lock [object=o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer@148b207a, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.waitCheckpointEvent(Checkpointer.java:820) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:260) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#94", id=138, state=WAITING, blockCnt=1, waitCnt=547] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64b555e, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#95", id=139, state=WAITING, blockCnt=0, waitCnt=900] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64b555e, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#96", id=140, state=WAITING, blockCnt=3, waitCnt=8578] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4cd91aba, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#97", id=141, state=WAITING, blockCnt=4, waitCnt=8412] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4cd91aba, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-thread", id=142, state=BLOCKED, blockCnt=3, waitCnt=5688302] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11657) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11559) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11539) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction$$Lambda$940/0x0000000800717c40.run(Unknown Source) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="disco-notifier-worker-#98", id=143, state=WAITING, blockCnt=0, waitCnt=4] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@318dd7e0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2830) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2875) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-1", id=144, state=WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="disco-event-worker-#99", id=145, state=WAITING, blockCnt=0, waitCnt=3] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@69b84eec, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:3089) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:3060) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-2", id=146, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-3", id=147, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-worker-#100", id=148, state=BLOCKED, blockCnt=5, waitCnt=5658039] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@578c6d85 Thread [name="defragmentation-worker-#101", id=149, state=BLOCKED, blockCnt=6, waitCnt=5669861] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@3e0c2a43 Thread [name="defragmentation-worker-#102", id=150, state=BLOCKED, blockCnt=6, waitCnt=5677051] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6930c0bf Thread [name="defragmentation-worker-#103", id=151, state=RUNNABLE, blockCnt=7, waitCnt=5680886] at java.management@11.0.18/sun.management.ThreadImpl.dumpThreads0(Native Method) at java.management@11.0.18/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:521) at java.management@11.0.18/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:509) at app//o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1529) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:205) - locked o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3 at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@4a151ef5 Thread [name="defragmentation-worker-#104", id=152, state=BLOCKED, blockCnt=6, waitCnt=5686371] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5b23f81a Thread [name="defragmentation-worker-#105", id=153, state=BLOCKED, blockCnt=5, waitCnt=5681521] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@f006ffb Thread [name="defragmentation-worker-#106", id=154, state=BLOCKED, blockCnt=5, waitCnt=5685720] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@30bd395b Thread [name="defragmentation-worker-#107", id=155, state=BLOCKED, blockCnt=6, waitCnt=5693569] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5092ba62 Thread [name="defragmentation-worker-#108", id=156, state=BLOCKED, blockCnt=3, waitCnt=5693927] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@2a626975 Thread [name="defragmentation-worker-#109", id=157, state=BLOCKED, blockCnt=1, waitCnt=5665547] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@381dcd99 Thread [name="defragmentation-worker-#110", id=158, state=BLOCKED, blockCnt=4, waitCnt=5673629] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@1558a7c0 Thread [name="defragmentation-worker-#111", id=159, state=BLOCKED, blockCnt=3, waitCnt=5702683] Lock [object=o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3, ownerName=defragmentation-worker-#103, ownerId=151] at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:167) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:207) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@771d2a0b Thread [name="exchange-worker-#112", id=160, state=TIMED_WAITING, blockCnt=0, waitCnt=148] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@504eb410, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) at java.base@11.0.18/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:3253) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:3182) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-client-listener-#113", id=161, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@fa35b2e - locked sun.nio.ch.EPollSelectorImpl@66d30625 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-0-#51", id=78, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4db7eb3d - locked sun.nio.ch.EPollSelectorImpl@687c28af at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-1-#52", id=79, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7d9364a5 - locked sun.nio.ch.EPollSelectorImpl@38fa71c at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-2-#53", id=80, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@71135a53 - locked sun.nio.ch.EPollSelectorImpl@78232d8e at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-3-#54", id=81, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1e73c863 - locked sun.nio.ch.EPollSelectorImpl@603a3bf6 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-4-#55", id=82, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@b902460 - locked sun.nio.ch.EPollSelectorImpl@6a3e27c1 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-5-#56", id=83, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6a8eca35 - locked sun.nio.ch.EPollSelectorImpl@5b5ab919 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-6-#57", id=84, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@51e33536 - locked sun.nio.ch.EPollSelectorImpl@4a7800e4 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-7-#58", id=85, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@563bb4fc - locked sun.nio.ch.EPollSelectorImpl@58a74d53 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="services-deployment-worker-#114", id=162, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@102599f4, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.service.ServiceDeploymentManager$ServicesDeploymentWorker.body(ServiceDeploymentManager.java:466) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="session-timeout-worker-#59", id=86, state=TIMED_WAITING, blockCnt=0, waitCnt=738] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.processors.rest.GridRestProcessor$4.body(GridRestProcessor.java:504) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#115", id=173, state=WAITING, blockCnt=0, waitCnt=18484583] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ad0c434, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#116", id=174, state=WAITING, blockCnt=0, waitCnt=18517503] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ad0c434, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-22", id=231, state=TIMED_WAITING, blockCnt=25, waitCnt=5957027] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-35", id=291, state=TIMED_WAITING, blockCnt=11, waitCnt=1230820] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#199", id=292, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#200", id=293, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="stat-mgmt-#201", id=294, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@73681568, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#202", id=295, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#203", id=296, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#204", id=297, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#205", id=298, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#206", id=299, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) [2023-07-06T13:21:18,265][WARN ][defragmentation-worker-#103][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:21:18,266][WARN ][defragmentation-worker-#108][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,266][INFO ][defragmentation-worker-#108][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,265][ERROR][defragmentation-worker-#103][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,276][WARN ][defragmentation-worker-#108][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:21:18,276][ERROR][defragmentation-worker-#108][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,276][WARN ][defragmentation-worker-#105][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,277][INFO ][defragmentation-worker-#105][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,287][WARN ][defragmentation-worker-#105][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:21:18,287][ERROR][defragmentation-worker-#105][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,288][WARN ][defragmentation-worker-#106][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,288][INFO ][defragmentation-worker-#106][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,302][WARN ][defragmentation-worker-#106][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:21:18,302][WARN ][defragmentation-worker-#110][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,302][ERROR][defragmentation-worker-#106][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,303][INFO ][defragmentation-worker-#110][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,312][WARN ][defragmentation-worker-#110][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:21:18,313][ERROR][defragmentation-worker-#110][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,313][WARN ][defragmentation-thread][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11657) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11559) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11539) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,313][INFO ][defragmentation-thread][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,323][WARN ][defragmentation-thread][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:21:18,325][WARN ][defragmentation-worker-#107][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,323][ERROR][defragmentation-thread][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11657) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11559) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.doInParallel(IgniteUtils.java:11539) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,325][INFO ][defragmentation-worker-#107][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,336][WARN ][defragmentation-worker-#107][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#103, id=151], state=RUNNABLE Locked pages = [282368329909939[000100d0000006b3](r=1|w=0)] Locked pages log: name=defragmentation-worker-#103 time=(1688638878325, 2023-07-06 13:21:18.325) L=1 -> Read lock pageId=282368329909939, structureId=link-map [pageIdHex=000100d0000006b3, partId=208, pageIdx=1715, flags=00000001] [2023-07-06T13:21:18,336][ERROR][defragmentation-worker-#107][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,336][WARN ][defragmentation-worker-#100][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,337][INFO ][defragmentation-worker-#100][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,348][WARN ][defragmentation-worker-#100][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#105, id=153], state=RUNNABLE Locked pages = [282346855074321[000100cb00000a11](r=0|w=1)] Locked pages log: name=defragmentation-worker-#105 time=(1688638878337, 2023-07-06 13:21:18.337) L=1 -> Write lock pageId=282346855074321, structureId=link-map [pageIdHex=000100cb00000a11, partId=203, pageIdx=2577, flags=00000001] [2023-07-06T13:21:18,348][ERROR][defragmentation-worker-#100][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,348][WARN ][defragmentation-worker-#111][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,349][INFO ][defragmentation-worker-#111][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,367][WARN ][defragmentation-worker-#111][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#107, id=155], state=RUNNABLE Locked pages = [282342560172512[000100ca000109e0](r=0|w=1)] Locked pages log: name=defragmentation-worker-#107 time=(1688638878349, 2023-07-06 13:21:18.349) L=1 -> Write lock pageId=282342560172512, structureId=cache0-202 [pageIdHex=000100ca000109e0, partId=202, pageIdx=68064, flags=00000001] -> Try Write lock nextOpPageId=1126767490304264, nextOpStructureId=cache0-202 [pageIdHex=000400ca00010908, partId=202, pageIdx=67848, flags=00000004] [2023-07-06T13:21:18,367][ERROR][defragmentation-worker-#111][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,367][WARN ][defragmentation-worker-#101][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,368][INFO ][defragmentation-worker-#101][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,378][WARN ][defragmentation-worker-#101][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:21:18,378][WARN ][defragmentation-worker-#102][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,378][ERROR][defragmentation-worker-#101][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,379][INFO ][defragmentation-worker-#102][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,389][WARN ][defragmentation-worker-#102][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#111, id=159], state=RUNNABLE Locked pages = [] Locked pages log: name=defragmentation-worker-#111 time=(1688638878379, 2023-07-06 13:21:18.379) -> Try Read lock nextOpPageId=1126763195272522, nextOpStructureId=cache0-p-201##CacheData [pageIdHex=000400c900000d4a, partId=201, pageIdx=3402, flags=00000004] [2023-07-06T13:21:18,390][ERROR][defragmentation-worker-#102][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,390][WARN ][defragmentation-worker-#104][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,390][INFO ][defragmentation-worker-#104][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,401][WARN ][defragmentation-worker-#104][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#103, id=151], state=RUNNABLE Locked pages = [282368329962101[000100d00000d275](r=0|w=1)] Locked pages log: name=defragmentation-worker-#103 time=(1688638878390, 2023-07-06 13:21:18.390) L=1 -> Write lock pageId=282368329962101, structureId=cache0-208 [pageIdHex=000100d00000d275, partId=208, pageIdx=53877, flags=00000001] Thread=[name=defragmentation-worker-#111, id=159], state=RUNNABLE Locked pages = [282338265215544[000100c900013238](r=0|w=1)] Locked pages log: name=defragmentation-worker-#111 time=(1688638878390, 2023-07-06 13:21:18.390) L=1 -> Write lock pageId=282338265215544, structureId=cache0-201 [pageIdHex=000100c900013238, partId=201, pageIdx=78392, flags=00000001] [2023-07-06T13:21:18,401][ERROR][defragmentation-worker-#104][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,402][WARN ][defragmentation-worker-#109][] Possible failure suppressed accordingly to a configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=SYSTEM_CRITICAL_OPERATION_TIMEOUT, err=class o.a.i.IgniteException: Checkpoint read lock acquisition has been timed out.]] org.apache.ignite.IgniteException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:205) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:18,402][INFO ][defragmentation-worker-#109][FailureProcessor] Thread dump is hidden due to throttling settings. Set IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT property to 0 to see all thread dumps. [2023-07-06T13:21:18,413][WARN ][defragmentation-worker-#109][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#102, id=150], state=RUNNABLE Locked pages = [282364035000316[000100cf0000e7fc](r=0|w=1)] Locked pages log: name=defragmentation-worker-#102 time=(1688638878402, 2023-07-06 13:21:18.402) L=1 -> Write lock pageId=282364035000316, structureId=cache0-207 [pageIdHex=000100cf0000e7fc, partId=207, pageIdx=59388, flags=00000001] Thread=[name=defragmentation-worker-#106, id=154], state=RUNNABLE Locked pages = [282333970272361[000100c800019069](r=0|w=1)] Locked pages log: name=defragmentation-worker-#106 time=(1688638878402, 2023-07-06 13:21:18.402) L=1 -> Write lock pageId=282333970272361, structureId=cache0-200 [pageIdHex=000100c800019069, partId=200, pageIdx=102505, flags=00000001] Thread=[name=defragmentation-worker-#110, id=158], state=RUNNABLE Locked pages = [] Locked pages log: name=defragmentation-worker-#110 time=(1688638878402, 2023-07-06 13:21:18.402) -> Try Read lock nextOpPageId=1126780375201285, nextOpStructureId=cache0-p-205##CacheData [pageIdHex=000400cd0000f605, partId=205, pageIdx=62981, flags=00000004] [2023-07-06T13:21:18,413][ERROR][defragmentation-worker-#109][CheckpointTimeoutLock] Checkpoint read lock acquisition has been timed out. org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock$CheckpointReadLockTimeoutException: Checkpoint read lock acquisition has been timed out. at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.failCheckpointReadLock(CheckpointTimeoutLock.java:210) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:108) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:21:24,906][WARN ][defragmentation-worker-#105][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragPartitionsDataRegion [2023-07-06T13:21:25,509][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=8936ms, pages=18542692, reason='partition defragmented'] [2023-07-06T13:21:33,896][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=10ms, reason='timeout'] [2023-07-06T13:23:34,608][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=10ms, reason='timeout'] [2023-07-06T13:24:22,814][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=1ms, checkpointLockHoldTime=11ms, reason='timeout'] [2023-07-06T13:26:15,683][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=14ms, reason='timeout'] [2023-07-06T13:27:19,326][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=11ms, reason='timeout'] [2023-07-06T13:28:10,032][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=12ms, reason='timeout'] [2023-07-06T13:28:58,188][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=16ms, reason='timeout'] [2023-07-06T13:29:59,811][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=13ms, reason='timeout'] [2023-07-06T13:30:55,214][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=17ms, reason='timeout'] [2023-07-06T13:31:46,966][ERROR][db-checkpoint-thread-defrag-#69][] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.IgniteCheckedException: Compound exception for CountDownFuture.]] org.apache.ignite.IgniteCheckedException: Compound exception for CountDownFuture. at org.apache.ignite.internal.util.future.CountDownFuture.addError(CountDownFuture.java:72) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.future.CountDownFuture.onDone(CountDownFuture.java:46) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.future.CountDownFuture.onDone(CountDownFuture.java:28) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:543) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointPagesWriter.run(CheckpointPagesWriter.java:167) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:829) [?:?] Suppressed: java.lang.AssertionError: Invalid state. Type is 0! pageId = 000100b700001eb8 at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.copyPageForCheckpoint(PageMemoryImpl.java:1359) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.checkpointWritePage(PageMemoryImpl.java:1277) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointPagesWriter.writePages(CheckpointPagesWriter.java:208) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointPagesWriter.run(CheckpointPagesWriter.java:150) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-06T13:31:46,967][ERROR][db-checkpoint-thread-defrag-#69][FailureProcessor] No deadlocked threads detected. [2023-07-06T13:31:48,735][WARN ][jvm-pause-detector-worker][IgniteKernal] Possible too long JVM pause: 1730 milliseconds. [2023-07-06T13:31:48,738][ERROR][db-checkpoint-thread-defrag-#69][FailureProcessor] Thread dump at 2023/07/06 13:31:48 MSK Thread [name="main", id=1, state=WAITING, blockCnt=8, waitCnt=17272] Lock [object=java.util.concurrent.CountDownLatch$Sync@164473cb, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345) at java.base@11.0.18/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232) at app//o.a.i.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:398) Locked synchronizers: java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@57725775 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@36b8db00 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@4fb8b8c5 Thread [name="Reference Handler", id=2, state=RUNNABLE, blockCnt=3, waitCnt=0] at java.base@11.0.18/java.lang.ref.Reference.waitForReferencePendingList(Native Method) at java.base@11.0.18/java.lang.ref.Reference.processPendingReferences(Reference.java:241) at java.base@11.0.18/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:213) Thread [name="Finalizer", id=3, state=WAITING, blockCnt=4, waitCnt=5] Lock [object=java.lang.ref.ReferenceQueue$Lock@6654dad8, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at java.base@11.0.18/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:170) Thread [name="Signal Dispatcher", id=4, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Common-Cleaner", id=19, state=TIMED_WAITING, blockCnt=18, waitCnt=37] Lock [object=java.lang.ref.ReferenceQueue$Lock@2358f098, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at java.base@11.0.18/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.18/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) at java.base@11.0.18/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:161) Thread [name="Log4j2-TF-6-Scheduled-2", id=23, state=TIMED_WAITING, blockCnt=0, waitCnt=24] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@29ac1aa6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="jvm-pause-detector-worker", id=24, state=TIMED_WAITING, blockCnt=0, waitCnt=27419] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:124) Thread [name="ignite-clock", id=25, state=TIMED_WAITING, blockCnt=0, waitCnt=135582] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils$25.run(IgniteUtils.java:3641) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-0-#1", id=26, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-1-#2", id=27, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-2-#3", id=28, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-3-#4", id=29, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-4-#5", id=30, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-5-#6", id=31, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-6-#7", id=32, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-7-#8", id=33, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-8-#9", id=34, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-9-#10", id=35, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-10-#11", id=36, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-11-#12", id=37, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-12-#13", id=38, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-13-#14", id=39, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-14-#15", id=40, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-15-#16", id=41, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-0-#17", id=42, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-1-#18", id=43, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-2-#19", id=44, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-3-#20", id=45, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-4-#21", id=46, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-5-#22", id=47, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-6-#23", id=48, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-7-#24", id=49, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-8-#25", id=50, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-9-#26", id=51, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-10-#27", id=52, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-11-#28", id=53, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-12-#29", id=54, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-13-#30", id=55, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-14-#31", id=56, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-15-#32", id=57, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-timeout-worker-#38", id=63, state=TIMED_WAITING, blockCnt=3, waitCnt=7342] Lock [object=java.lang.Object@1cc3fe73, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:269) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-comm-#47%TcpCommunicationSpi%", id=72, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@5c016529 - locked sun.nio.ch.EPollSelectorImpl@38398201 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-0-#39%TcpCommunicationSpi%", id=64, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7189c68a - locked sun.nio.ch.EPollSelectorImpl@72ba9eae at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-1-#40%TcpCommunicationSpi%", id=65, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@25c595fd - locked sun.nio.ch.EPollSelectorImpl@519832ce at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-2-#41%TcpCommunicationSpi%", id=66, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6e1a13e1 - locked sun.nio.ch.EPollSelectorImpl@48f3bf62 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-3-#42%TcpCommunicationSpi%", id=67, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4190b1fc - locked sun.nio.ch.EPollSelectorImpl@510ee542 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-4-#43%TcpCommunicationSpi%", id=68, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6f357199 - locked sun.nio.ch.EPollSelectorImpl@3a68cbb2 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-5-#44%TcpCommunicationSpi%", id=69, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7e339252 - locked sun.nio.ch.EPollSelectorImpl@58315c23 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-6-#45%TcpCommunicationSpi%", id=70, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1ccc4705 - locked sun.nio.ch.EPollSelectorImpl@60053a75 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-7-#46%TcpCommunicationSpi%", id=71, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2c69525 - locked sun.nio.ch.EPollSelectorImpl@5044011a at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="tcp-comm-worker-#1-#48", id=73, state=TIMED_WAITING, blockCnt=0, waitCnt=3] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e4e9078, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at app//o.a.i.spi.communication.tcp.internal.CommunicationWorker.body(CommunicationWorker.java:167) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at app//o.a.i.spi.communication.tcp.TcpCommunicationSpi$3.body(TcpCommunicationSpi.java:848) at app//o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:58) Thread [name="binary-metadata-writer-#49", id=75, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@67d1c156, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body0(BinaryMetadataFileStore.java:477) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body(BinaryMetadataFileStore.java:458) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="page-lock-tracker-timeout", id=76, state=TIMED_WAITING, blockCnt=0, waitCnt=24] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.worker.CycleThread.run(CycleThread.java:47) Thread [name="long-qry-#50", id=77, state=TIMED_WAITING, blockCnt=0, waitCnt=1386] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:8345) at app//o.a.i.i.processors.query.running.HeavyQueriesTracker$1.body(HeavyQueriesTracker.java:104) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-rest-#64", id=91, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@51880f34 - locked sun.nio.ch.EPollSelectorImpl@21b488ce at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-0-#60", id=87, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7ba301d0 - locked sun.nio.ch.EPollSelectorImpl@1211a483 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-1-#61", id=88, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@371f85a - locked sun.nio.ch.EPollSelectorImpl@686e202 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-2-#62", id=89, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6548de83 - locked sun.nio.ch.EPollSelectorImpl@7ae24b15 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-3-#63", id=90, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@765ddf3b - locked sun.nio.ch.EPollSelectorImpl@477ab500 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-data-loader-flusher-#68", id=95, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6465d7aa, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.DelayQueue.take(DelayQueue.java:217) at app//o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java:106) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="db-checkpoint-thread-defrag-#69", id=96, state=RUNNABLE, blockCnt=8, waitCnt=15] at java.management@11.0.18/sun.management.ThreadImpl.dumpThreads0(Native Method) at java.management@11.0.18/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:521) at java.management@11.0.18/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:509) at app//o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1529) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:205) - locked o.a.i.i.processors.failure.FailureProcessor@7cc8a0f3 at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.doCheckpoint(Checkpointer.java:508) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:278) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="ttl-cleanup-worker-#73", id=101, state=WAITING, blockCnt=1, waitCnt=2] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:142) at app//o.a.i.i.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:164) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-5", id=121, state=TIMED_WAITING, blockCnt=72, waitCnt=18859897] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="db-checkpoint-thread-#93", id=137, state=TIMED_WAITING, blockCnt=0, waitCnt=105] Lock [object=o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer@148b207a, ownerName=null, ownerId=-1] at java.base@11.0.18/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.waitCheckpointEvent(Checkpointer.java:820) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:260) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#94", id=138, state=WAITING, blockCnt=1, waitCnt=599] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64b555e, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#95", id=139, state=WAITING, blockCnt=0, waitCnt=953] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64b555e, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#96", id=140, state=WAITING, blockCnt=3, waitCnt=8578] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4cd91aba, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#97", id=141, state=WAITING, blockCnt=4, waitCnt=8412] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4cd91aba, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-thread", id=142, state=WAITING, blockCnt=5, waitCnt=7717628] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11657) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11559) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11539) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction$$Lambda$940/0x0000000800717c40.run(Unknown Source) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="disco-notifier-worker-#98", id=143, state=WAITING, blockCnt=0, waitCnt=4] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@318dd7e0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2830) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2875) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-1", id=144, state=WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="disco-event-worker-#99", id=145, state=WAITING, blockCnt=0, waitCnt=3] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@69b84eec, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:3089) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:3060) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-2", id=146, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-3", id=147, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6db0852a, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-worker-#100", id=148, state=WAITING, blockCnt=5, waitCnt=7674958] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@578c6d85 Thread [name="defragmentation-worker-#101", id=149, state=WAITING, blockCnt=8, waitCnt=7686453] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@3e0c2a43 Thread [name="defragmentation-worker-#102", id=150, state=WAITING, blockCnt=6, waitCnt=7703865] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6930c0bf Thread [name="defragmentation-worker-#103", id=151, state=WAITING, blockCnt=8, waitCnt=7718667] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@4a151ef5 Thread [name="defragmentation-worker-#104", id=152, state=WAITING, blockCnt=6, waitCnt=7706522] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5b23f81a Thread [name="defragmentation-worker-#105", id=153, state=WAITING, blockCnt=5, waitCnt=7704479] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@f006ffb Thread [name="defragmentation-worker-#106", id=154, state=WAITING, blockCnt=7, waitCnt=7709685] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@30bd395b Thread [name="defragmentation-worker-#107", id=155, state=WAITING, blockCnt=6, waitCnt=7717993] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5092ba62 Thread [name="defragmentation-worker-#108", id=156, state=WAITING, blockCnt=3, waitCnt=7725813] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@2a626975 Thread [name="defragmentation-worker-#109", id=157, state=WAITING, blockCnt=1, waitCnt=7675329] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@381dcd99 Thread [name="defragmentation-worker-#110", id=158, state=WAITING, blockCnt=4, waitCnt=7695771] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@1558a7c0 Thread [name="defragmentation-worker-#111", id=159, state=WAITING, blockCnt=3, waitCnt=7731227] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointTimeoutLock.checkpointReadLock(CheckpointTimeoutLock.java:144) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:724) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1061/0x0000000800746840.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$996/0x0000000800735440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11638) at app//o.a.i.i.util.IgniteUtils$$Lambda$1012/0x000000080073a040.call(Unknown Source) at java.base@11.0.18/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@771d2a0b Thread [name="exchange-worker-#112", id=160, state=TIMED_WAITING, blockCnt=0, waitCnt=274] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@504eb410, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) at java.base@11.0.18/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:3253) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:3182) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-client-listener-#113", id=161, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@fa35b2e - locked sun.nio.ch.EPollSelectorImpl@66d30625 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-0-#51", id=78, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4db7eb3d - locked sun.nio.ch.EPollSelectorImpl@687c28af at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-1-#52", id=79, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7d9364a5 - locked sun.nio.ch.EPollSelectorImpl@38fa71c at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-2-#53", id=80, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@71135a53 - locked sun.nio.ch.EPollSelectorImpl@78232d8e at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-3-#54", id=81, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1e73c863 - locked sun.nio.ch.EPollSelectorImpl@603a3bf6 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-4-#55", id=82, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@b902460 - locked sun.nio.ch.EPollSelectorImpl@6a3e27c1 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-5-#56", id=83, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6a8eca35 - locked sun.nio.ch.EPollSelectorImpl@5b5ab919 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-6-#57", id=84, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@51e33536 - locked sun.nio.ch.EPollSelectorImpl@4a7800e4 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-7-#58", id=85, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.18/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.18/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@563bb4fc - locked sun.nio.ch.EPollSelectorImpl@58a74d53 at java.base@11.0.18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="services-deployment-worker-#114", id=162, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@102599f4, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.service.ServiceDeploymentManager$ServicesDeploymentWorker.body(ServiceDeploymentManager.java:466) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="session-timeout-worker-#59", id=86, state=TIMED_WAITING, blockCnt=0, waitCnt=1367] at java.base@11.0.18/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.processors.rest.GridRestProcessor$4.body(GridRestProcessor.java:504) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#115", id=173, state=WAITING, blockCnt=0, waitCnt=36865689] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ad0c434, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#116", id=174, state=WAITING, blockCnt=0, waitCnt=18748497] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1ad0c434, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-44", id=309, state=TIMED_WAITING, blockCnt=1, waitCnt=10189322] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#285", id=399, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#286", id=400, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#287", id=401, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#288", id=402, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#289", id=403, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="stat-mgmt-#290", id=404, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@73681568, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#291", id=405, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#292", id=406, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11471fc6, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.18/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-48", id=407, state=TIMED_WAITING, blockCnt=0, waitCnt=12194] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@cb782d0, ownerName=null, ownerId=-1] at java.base@11.0.18/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.18/java.lang.Thread.run(Thread.java:829) [2023-07-06T13:31:48,750][WARN ][db-checkpoint-thread-defrag-#69][CacheDiagnosticManager] Page locks dump: [2023-07-06T13:31:48,751][ERROR][db-checkpoint-thread-defrag-#69][] JVM will be halted immediately due to the failure: [failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.IgniteCheckedException: Compound exception for CountDownFuture.]]