[2023-07-05T16:03:28,222][WARN ][main][] Property autoActivation deprecated. Use clusterStateOnStart instead. [2023-07-05T16:03:28,374][INFO ][main][IgniteKernal] >>> __________ ________________ >>> / _/ ___/ |/ / _/_ __/ __/ >>> _/ // (7 7 // / / / / _/ >>> /___/\___/_/|_/___/ /_/ /___/ >>> >>> ver. 2.16.0-SNAPSHOT#20230704-sha1:a998fe93 >>> 2023 Copyright(C) Apache Software Foundation >>> >>> Ignite documentation: https://ignite.apache.org [2023-07-05T16:03:28,378][INFO ][main][IgniteKernal] Config URL: file:/test/nao/server/config/default-config.xml [2023-07-05T16:03:28,403][INFO ][main][IgniteKernal] IgniteConfiguration [igniteInstanceName=null, pubPoolSize=16, svcPoolSize=16, callbackPoolSize=16, stripedPoolSize=16, sysPoolSize=16, mgmtPoolSize=4, dataStreamerPoolSize=16, utilityCachePoolSize=16, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=16, buildIdxPoolSize=4, igniteHome=/test/nao/server, igniteWorkDir=/test/nao/server/work, mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@51bd8b5c, nodeId=2e17ec7e-ac14-421c-9feb-5dae2d251e0a, marsh=BinaryMarshaller [], marshLocJobs=false, p2pEnabled=true, netTimeout=5000, netCompressionLevel=1, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, addressFilter=null, sockTimeout=0, ackTimeout=0, marsh=null, reconCnt=10, reconDelay=2000, maxAckTimeout=600000, soLinger=0, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null, skipAddrsRandomization=false], segPlc=USE_FAILURE_HANDLER, segResolveAttempts=2, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=10000, commSpi=TcpCommunicationSpi [connectGate=org.apache.ignite.spi.communication.tcp.internal.ConnectGateway@7186333e, ctxInitLatch=java.util.concurrent.CountDownLatch@692fd26[Count = 1], stopping=false, clientPool=null, nioSrvWrapper=null, stateProvider=null], evtSpi=org.apache.ignite.spi.eventstorage.NoopEventStorageSpi@36f1046f, colSpi=NoopCollisionSpi [], deploySpi=LocalDeploymentSpi [], indexingSpi=org.apache.ignite.spi.indexing.noop.NoopIndexingSpi@61e3cf4d, addrRslvr=null, encryptionSpi=org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi@3cec79d3, tracingSpi=org.apache.ignite.spi.tracing.NoopTracingSpi@64b70919, clientMode=false, rebalanceThreadPoolSize=4, rebalanceTimeout=10000, rebalanceBatchesPrefetchCnt=3, rebalanceThrottle=0, rebalanceBatchSize=524288, txCfg=TransactionConfiguration [txSerEnabled=false, dfltIsolation=READ_COMMITTED, dfltConcurrency=PESSIMISTIC, dfltTxTimeout=300000, txTimeoutOnPartitionMapExchange=10000, deadlockTimeout=10000, pessimisticTxLogSize=0, pessimisticTxLogLinger=10000, tmLookupClsName=null, txManagerFactory=null, useJtaSync=false], cacheSanityCheckEnabled=true, discoStartupDelay=60000, deployMode=SHARED, p2pMissedCacheSize=100, locHost=null, timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=30000, sysWorkerBlockedTimeout=null, clientFailureDetectionTimeout=30000, metricsLogFreq=0, connectorCfg=ConnectorConfiguration [jettyPath=null, host=0.0.0.0, port=11211, noDelay=true, directBuf=false, sndBufSize=32768, rcvBufSize=32768, idleQryCurTimeout=600000, idleQryCurCheckFreq=60000, sndQueueLimit=0, selectorCnt=4, idleTimeout=180000, sslEnabled=false, sslClientAuth=false, sslFactory=null, portRange=100, threadPoolSize=16, msgInterceptor=null], odbcCfg=null, warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000, cacheMode=PARTITIONED, backups=1, aff=null, grpName=null], classLdr=null, sslCtxFactory=null, platformCfg=null, binaryCfg=null, memCfg=null, pstCfg=null, dsCfg=DataStorageConfiguration [pageSize=1024, concLvl=0, sysDataRegConf=org.apache.ignite.configuration.SystemDataRegionConfiguration@28348c6, dfltDataRegConf=DataRegionConfiguration [name=default, maxSize=6442450944, initSize=268435456, swapPath=null, pageEvictionMode=DISABLED, pageReplacementMode=CLOCK, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=true, checkpointPageBufSize=0, lazyMemoryAllocation=true, warmUpCfg=null, memoryAllocator=null, cdcEnabled=false], dataRegions=null, storagePath=/test/nao/server/work/data, checkpointFreq=60000, lockWaitTime=10000, checkpointThreads=2, checkpointWriteOrder=SEQUENTIAL, walHistSize=20, maxWalArchiveSize=10737418240, walSegments=10, walSegmentSize=67108864, walPath=/test/nao/server/work/wal, walArchivePath=/test/nao/server/work/wal_archive, cdcWalPath=db/wal/cdc, cdcWalDirMaxSize=0, metricsEnabled=false, walMode=LOG_ONLY, walTlbSize=131072, walBuffSize=0, walFlushFreq=2000, walFsyncDelay=1000, walRecordIterBuffSize=67108864, alwaysWriteFullPages=false, fileIOFactory=org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory@23202c31, metricsSubIntervalCnt=5, metricsRateTimeInterval=60000, walAutoArchiveAfterInactivity=-1, walForceArchiveTimeout=-1, writeThrottlingEnabled=false, walCompactionEnabled=false, walCompactionLevel=1, checkpointReadLockTimeout=30000, walPageCompression=DISABLED, walPageCompressionLevel=null, dfltWarmUpCfg=null, encCfg=org.apache.ignite.configuration.EncryptionConfiguration@b016b4e, defragmentationThreadPoolSize=16, minWalArchiveSize=-1, memoryAllocator=null], snapshotPath=/test/nao/backup/snapshot, snapshotThreadPoolSize=10, activeOnStart=true, activeOnStartPropSetFlag=false, autoActivation=false, autoActivationPropSetFlag=true, clusterStateOnStart=null, sqlConnCfg=null, cliConnCfg=ClientConnectorConfiguration [host=0.0.0.0, port=10800, portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true, maxOpenCursorsPerConn=128, threadPoolSize=16, selectorCnt=8, idleTimeout=0, handshakeTimeout=10000, jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false, useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null, thinCliCfg=ThinClientConfiguration [maxActiveTxPerConn=100, maxActiveComputeTasksPerConn=100, sendServerExcStackTraceToClient=false]], mvccVacuumThreadCnt=2, mvccVacuumFreq=5000, authEnabled=false, failureHnd=null, commFailureRslvr=null, sqlCfg=SqlConfiguration [longQryWarnTimeout=3000, dfltQryTimeout=0, sqlQryHistSize=1000, validationEnabled=false], asyncContinuationExecutor=null] [2023-07-05T16:03:28,404][INFO ][main][IgniteKernal] OS: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 [2023-07-05T16:03:28,405][INFO ][main][IgniteKernal] OS user: admin [2023-07-05T16:03:28,408][INFO ][main][IgniteKernal] PID: 19520 [2023-07-05T16:03:28,409][INFO ][main][IgniteKernal] Language runtime: Java Platform API Specification ver. 11 [2023-07-05T16:03:28,409][INFO ][main][IgniteKernal] VM information: OpenJDK Runtime Environment 11.0.17+8 Eclipse Adoptium OpenJDK 64-Bit Server VM 11.0.17+8 [2023-07-05T16:03:28,412][INFO ][main][IgniteKernal] VM total memory: 2.0GB [2023-07-05T16:03:28,412][INFO ][main][IgniteKernal] Remote Management [restart: on, REST: on, JMX (remote: off)] [2023-07-05T16:03:28,412][INFO ][main][IgniteKernal] Logger: Log4J2Logger [quiet=false, config=/test/nao/server/config/ignite-log4j2.xml] [2023-07-05T16:03:28,415][INFO ][main][IgniteKernal] System cache's DataRegion size is configured to 40 MB. Use DataStorageConfiguration.systemRegionInitialSize property to change the setting. [2023-07-05T16:03:28,416][INFO ][main][IgniteKernal] Configured caches [in 'sysMemPlc' dataRegion: ['ignite-sys-cache']] [2023-07-05T16:03:28,416][WARN ][main][IgniteKernal] Peer class loading is enabled (disable it in production for performance and deployment consistency reasons) [2023-07-05T16:03:28,421][INFO ][main][IgniteKernal] 3-rd party licenses can be found at: /test/nao/server/libs/licenses [2023-07-05T16:03:28,501][INFO ][main][IgnitePluginProcessor] Configured plugins: [2023-07-05T16:03:28,502][INFO ][main][IgnitePluginProcessor] ^-- None [2023-07-05T16:03:28,502][INFO ][main][IgnitePluginProcessor] [2023-07-05T16:03:28,506][INFO ][main][FailureProcessor] Configured failure handler: [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]]] [2023-07-05T16:03:28,773][INFO ][main][TcpCommunicationSpi] Successfully bound communication NIO server to TCP port [port=47900, locHost=/10.40.6.208, selectorsCnt=8, selectorSpins=0, pairedConn=false] [2023-07-05T16:03:28,773][WARN ][main][TcpCommunicationSpi] Message queue limit is set to 0 which may lead to potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and receiver sides. [2023-07-05T16:03:28,800][INFO ][main][GridCollisionManager] Collision resolution is disabled (all jobs will be activated upon arrival). [2023-07-05T16:03:28,844][INFO ][main][MaintenanceProcessor] Resolved store directory for node persistent data: /test/nao/server/work/data/ise_devmon_8 [2023-07-05T16:03:28,849][INFO ][main][IgniteKernal] Node is being started in maintenance mode. Starting IsolatedDiscoverySpi instead of configured discovery SPI. [2023-07-05T16:03:28,849][INFO ][main][IgniteKernal] Overriding 'clusterStateOnStart' configuration to 'INACTIVE'. [2023-07-05T16:03:28,874][INFO ][main][CacheObjectBinaryProcessorImpl] Resolved directory for serialized binary metadata: /test/nao/server/work/db/binary_meta/ise_devmon_8 [2023-07-05T16:03:28,910][WARN ][main][GridClusterStateProcessor] Property `autoActivation` will be ignored due to the property `clusterStateOnStart` is presented. [2023-07-05T16:03:29,040][INFO ][main][GridLocalConfigManager] Resolved page store work directory: /test/nao/server/work/data/ise_devmon_8 [2023-07-05T16:03:29,066][INFO ][main][FilePageStoreManager] Resolved page store work directory: /test/nao/server/work/data/ise_devmon_8 [2023-07-05T16:03:29,068][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log work directory: /test/nao/server/work/wal/ise_devmon_8 [2023-07-05T16:03:29,069][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log archive directory: /test/nao/server/work/wal_archive/ise_devmon_8 [2023-07-05T16:03:29,103][INFO ][main][FileHandleManagerImpl] Initialized write-ahead log manager [mode=LOG_ONLY] [2023-07-05T16:03:29,111][INFO ][main][GridCacheDatabaseSharedManager] Region size was reassigned by defragmentation reason: region = 'default', oldInitialSize = '268435456', newInitialSize = '268435456', oldMaxSize = '6442450944', newMaxSize = '2576980378 [2023-07-05T16:03:29,135][INFO ][main][GridCacheDatabaseSharedManager] Configured data regions initialized successfully [total=7] [2023-07-05T16:03:29,157][INFO ][main][IgniteSnapshotManager] Resolved snapshot work directory: /test/nao/backup/snapshot [2023-07-05T16:03:29,157][INFO ][main][IgniteSnapshotManager] Resolved temp directory for snapshot creation: /test/nao/server/work/data/ise_devmon_8/snp [2023-07-05T16:03:29,222][WARN ][main][IgniteH2Indexing] Serialization of Java objects in H2 was enabled. [2023-07-05T16:03:29,385][INFO ][main][ClientListenerProcessor] Client connector processor has started on TCP port 10800 [2023-07-05T16:03:29,439][INFO ][main][GridTcpRestProtocol] Command protocol successfully started [name=TCP binary, host=/0.0.0.0, port=11211] [2023-07-05T16:03:29,504][INFO ][main][IgniteKernal] Non-loopback local IPs: 10.40.6.208, 172.17.0.1 [2023-07-05T16:03:29,504][INFO ][main][IgniteKernal] Enabled local MACs: 0050560AF634, 024206B4ECF3 [2023-07-05T16:03:29,512][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688562199035-90288f0e-7e8f-4e18-8c07-77c77af3df2a-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688562199035-90288f0e-7e8f-4e18-8c07-77c77af3df2a-END.bin] [2023-07-05T16:03:29,526][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=89280, tableSize=7.4 MiB, replacementSize=11.0 KiB, checkpointBuffer=100.0 MiB] [2023-07-05T16:03:29,530][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=3223, fileOff=30466518, len=11697], lastMarked=WALPointer [idx=3223, fileOff=30466518, len=11697], lastCheckpointId=90288f0e-7e8f-4e18-8c07-77c77af3df2a] [2023-07-05T16:03:29,571][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@649b5891, actualFilePtr=WALPointer [idx=3223, fileOff=30478215, len=0]] [2023-07-05T16:03:29,599][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@7eaa2bc6, actualFilePtr=WALPointer [idx=3223, fileOff=30478215, len=0]] [2023-07-05T16:03:29,600][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=90288f0e-7e8f-4e18-8c07-77c77af3df2a, pos=WALPointer [idx=3223, fileOff=30466518, len=11697]] [2023-07-05T16:03:29,649][INFO ][main][GridCacheDatabaseSharedManager] Applying lost metastore updates since last checkpoint record [lastMarked=WALPointer [idx=3223, fileOff=30466518, len=11697], lastCheckpointId=90288f0e-7e8f-4e18-8c07-77c77af3df2a] [2023-07-05T16:03:29,674][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@7c0df4ab, actualFilePtr=WALPointer [idx=3223, fileOff=30478215, len=0]] [2023-07-05T16:03:29,682][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=30 ms] [2023-07-05T16:03:29,682][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-05T16:03:29,690][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=0, partitionsProcessed=0, time=0ms] [2023-07-05T16:03:29,730][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@6eda012b, actualFilePtr=WALPointer [idx=3223, fileOff=30478215, len=0]] [2023-07-05T16:03:29,770][INFO ][main][GridEncryptionManager] Encryption keys loaded from metastore. [grps=, masterKeyName=null] [2023-07-05T16:03:29,789][INFO ][main][GridClusterStateProcessor] Restoring history for BaselineTopology[id=0] [2023-07-05T16:03:29,817][INFO ][main][ClusterProcessor] Cluster ID and tag has been read from metastorage: ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-05T16:03:29,821][INFO ][main][IgniteClusterImpl] Shutdown policy was updated [oldVal=null, newVal=null] [2023-07-05T16:03:29,823][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustEnabled' was changed from 'null' to 'false' [2023-07-05T16:03:29,824][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustTimeout' was changed from 'null' to '300000' [2023-07-05T16:03:29,824][INFO ][main][IgniteTxManager] Transactions parameter 'txOwnerDumpRequestsAllowed' was changed from 'null' to 'true' [2023-07-05T16:03:29,824][INFO ][main][IgniteTxManager] Transactions parameter 'longOperationsDumpTimeout' was changed from 'null' to '60000' [2023-07-05T16:03:29,825][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpThreshold' was changed from 'null' to '0' [2023-07-05T16:03:29,826][INFO ][main][IgniteTxManager] Transactions parameter 'transactionTimeDumpSamplesCoefficient' was changed from 'null' to '0.0' [2023-07-05T16:03:29,826][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpSamplesPerSecondLimit' was changed from 'null' to '5' [2023-07-05T16:03:29,827][INFO ][main][IgniteTxManager] Transactions parameter 'collisionsDumpInterval' was changed from 'null' to '1000' [2023-07-05T16:03:29,827][INFO ][main][GridCacheDatabaseSharedManager] Historical rebalance WAL threshold changed [property=historical.rebalance.threshold, oldVal=null, newVal=500] [2023-07-05T16:03:29,828][INFO ][main][GridCacheDatabaseSharedManager] Checkpoint frequency deviation changed [oldVal=null, newVal=null] [2023-07-05T16:03:29,828][INFO ][main][IgniteSnapshotManager] The snapshot transfer rate is not limited. [2023-07-05T16:03:29,829][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.disabledFunctions' was changed from 'null' to '[FILE_WRITE, CANCEL_SESSION, MEMORY_USED, CSVREAD, LINK_SCHEMA, MEMORY_FREE, FILE_READ, CSVWRITE, SESSION_ID, LOCK_MODE]' [2023-07-05T16:03:29,829][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.defaultQueryTimeout' was changed from 'null' to '0' [2023-07-05T16:03:29,830][INFO ][main][IgniteStatisticsManagerImpl] Statistics usage state was changed from null to null [2023-07-05T16:03:29,847][INFO ][main][FilePageStoreManager] Cleanup cache stores [total=1, left=0, cleanFiles=false] [2023-07-05T16:03:29,965][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=3.2 GiB, pages=2962080, tableSize=244.0 MiB, replacementSize=361.6 KiB, checkpointBuffer=829.4 MiB] [2023-07-05T16:03:29,969][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=89280, tableSize=7.4 MiB, replacementSize=11.0 KiB, checkpointBuffer=100.0 MiB] [2023-07-05T16:03:29,971][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=89280, tableSize=7.4 MiB, replacementSize=11.0 KiB, checkpointBuffer=100.0 MiB] [2023-07-05T16:03:29,975][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=89280, tableSize=7.4 MiB, replacementSize=11.0 KiB, checkpointBuffer=100.0 MiB] [2023-07-05T16:03:29,989][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=368.6 MiB, pages=329120, tableSize=27.1 MiB, replacementSize=40.3 KiB, checkpointBuffer=256.0 MiB] [2023-07-05T16:03:29,990][INFO ][main][GridCacheDatabaseSharedManager] Data Regions Started: 7 [2023-07-05T16:03:29,991][INFO ][main][IgniteKernal] ^-- defragPartitionsDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=3317MB, maxCfg=3317MB, usedRam=0MB, freeRam=100%, allocRam=3317MB, allocTotal=0MB] ^-- sysMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- default region [type=default, persistence=true, lazyAlloc=true, ... initCfg=256MB, maxCfg=2457MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB] ^-- metastoreMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- TxLog region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- defragMappingDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=368MB, maxCfg=368MB, usedRam=0MB, freeRam=100%, allocRam=368MB, allocTotal=0MB] ^-- volatileDsMemPlc region [type=user, persistence=false, lazyAlloc=true, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=0MB] [2023-07-05T16:03:29,992][INFO ][main][GridCacheDatabaseSharedManager] Starting binary memory restore for: [-2100569601, -1368047378] [2023-07-05T16:03:30,063][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688562199035-90288f0e-7e8f-4e18-8c07-77c77af3df2a-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688562199035-90288f0e-7e8f-4e18-8c07-77c77af3df2a-END.bin] [2023-07-05T16:03:30,063][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=3223, fileOff=30466518, len=11697], lastMarked=WALPointer [idx=3223, fileOff=30466518, len=11697], lastCheckpointId=90288f0e-7e8f-4e18-8c07-77c77af3df2a] [2023-07-05T16:03:30,086][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@32a8ca06, actualFilePtr=WALPointer [idx=3223, fileOff=30478215, len=0]] [2023-07-05T16:03:30,113][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@1753475d, actualFilePtr=WALPointer [idx=3223, fileOff=30478215, len=0]] [2023-07-05T16:03:30,114][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=90288f0e-7e8f-4e18-8c07-77c77af3df2a, pos=WALPointer [idx=3223, fileOff=30466518, len=11697]] [2023-07-05T16:03:30,120][INFO ][main][GridCacheDatabaseSharedManager] Binary memory state restored at node startup [restoredPtr=WALPointer [idx=3223, fileOff=30478215, len=0]] [2023-07-05T16:03:30,126][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/nao/server/work/wal/ise_devmon_8/0000000000000003.wal, offset=30478215, ver=2] [2023-07-05T16:03:30,209][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=ignite-sys-cache, id=-2100569601, dataRegionName=sysMemPlc, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647, mvcc=false] [2023-07-05T16:03:30,296][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=2.4 GiB, pages=2194128, tableSize=180.7 MiB, replacementSize=267.9 KiB, checkpointBuffer=614.4 MiB] [2023-07-05T16:03:30,311][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=cache0, id=-1368047378, dataRegionName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647, mvcc=false] [2023-07-05T16:03:30,313][INFO ][main][GridCacheDatabaseSharedManager] Binary recovery performed in 320 ms. [2023-07-05T16:03:30,313][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688562199035-90288f0e-7e8f-4e18-8c07-77c77af3df2a-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688562199035-90288f0e-7e8f-4e18-8c07-77c77af3df2a-END.bin] [2023-07-05T16:03:30,314][INFO ][main][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=WALPointer [idx=3223, fileOff=30466518, len=11697], lastCheckpointId=90288f0e-7e8f-4e18-8c07-77c77af3df2a] [2023-07-05T16:03:30,337][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@1736c1e4, actualFilePtr=WALPointer [idx=3223, fileOff=30478244, len=0]] [2023-07-05T16:03:30,343][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=31 ms] [2023-07-05T16:03:30,343][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-05T16:03:32,761][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=2, partitionsProcessed=612, time=2s] [2023-07-05T16:03:32,765][INFO ][main][MaintenanceProcessor] Node requires maintenance, non-empty set of maintenance tasks is found: [defragmentationMaintenanceTask] [2023-07-05T16:03:32,768][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/nao/server/work/wal/ise_devmon_8/0000000000000003.wal, offset=30478244, ver=2] [2023-07-05T16:03:32,816][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint started [checkpointId=411ae41e-695a-4485-a358-06cc93ebabbf, startPtr=WALPointer [idx=3223, fileOff=31019428, len=11697], checkpointBeforeLockTime=22ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=3ms, checkpointLockHoldTime=6ms, walCpRecordFsyncDuration=4ms, writeCheckpointEntryDuration=11ms, splitAndSortCpPagesDuration=3ms, pages=512, reason='node started'] [2023-07-05T16:03:33,003][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint finished [cpId=411ae41e-695a-4485-a358-06cc93ebabbf, pages=512, markPos=WALPointer [idx=3223, fileOff=31019428, len=11697], walSegmentsCovered=[], markDuration=26ms, pagesWrite=26ms, fsync=158ms, total=232ms] [2023-07-05T16:03:33,014][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=5ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=3ms, checkpointLockHoldTime=4ms, reason='beforeDefragmentation'] [2023-07-05T16:03:33,018][INFO ][defragmentation-thread][CachePartitionDefragmentationManager] Defragmentation started. [2023-07-05T16:03:33,024][INFO ][defragmentation-thread][CacheGroupContext] Local state for group durability has changed [name=cache0, enabled=false] [2023-07-05T16:03:33,026][INFO ][disco-notifier-worker-#98][MvccProcessorImpl] Assigned mvcc coordinator [crd=MvccCoordinator [topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0], nodeId=2e17ec7e-ac14-421c-9feb-5dae2d251e0a, ver=1688562208851, local=true, initialized=false]] [2023-07-05T16:03:33,031][INFO ][sys-#86][ClusterProcessor] Writing cluster ID and tag to metastorage on ready for write ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-05T16:03:33,328][INFO ][main][IgniteKernal] Security status [authentication=off, sandbox=off, tls/ssl=off] [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] Performance suggestions for grid (fix if possible) [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] To disable, set -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] ^-- Set max direct memory size if getting 'OOME: Direct buffer memory' (add '-XX:MaxDirectMemorySize=[g|G|m|M|k|K]' to JVM options) [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] ^-- Speed up flushing of dirty pages by OS (alter vm.dirty_expire_centisecs parameter by setting to 500) [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] ^-- Reduce pages swapping ratio (set vm.swappiness=10 or less) [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] ^-- Disable assertions (remove '-ea' from JVM options) [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] Refer to this page for more performance suggestions: https://ignite.apache.org/docs/latest/perf-and-troubleshooting/memory-tuning [2023-07-05T16:03:33,329][INFO ][main][IgniteKernal] [2023-07-05T16:03:33,330][INFO ][main][IgniteKernal] Classpath value: /test/nao/server/libs/log4j-core-2.20.0.jar:/test/nao/server/libs/log4j-api-2.20.0.jar:/test/nao/server/libs/ignite-log4j2-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-core-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/annotations-16.0.3.jar:/test/nao/server/libs/cache-api-1.0.0.jar:/test/nao/server/libs/ignite-control-utility/ignite-control-utility-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-indexing/h2-1.4.197.jar:/test/nao/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar:/test/nao/server/libs/ignite-indexing/ignite-indexing-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-indexing/lucene-core-8.11.2.jar:/test/nao/server/libs/ignite-indexing/commons-codec-1.15.jar:/test/nao/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar:/test/nao/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/ignite-spring-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/commons-logging-1.1.1.jar:/test/nao/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar:/test/nao/server/libs/licenses/* [2023-07-05T16:03:33,330][INFO ][main][IgniteKernal] List of files containing in classpath: /test/nao/server/libs/log4j-core-2.20.0.jar;/test/nao/server/libs/log4j-api-2.20.0.jar;/test/nao/server/libs/ignite-log4j2-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-core-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/annotations-16.0.3.jar;/test/nao/server/libs/cache-api-1.0.0.jar;/test/nao/server/libs/ignite-control-utility/ignite-control-utility-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-indexing/h2-1.4.197.jar;/test/nao/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar;/test/nao/server/libs/ignite-indexing/ignite-indexing-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-indexing/lucene-core-8.11.2.jar;/test/nao/server/libs/ignite-indexing/commons-codec-1.15.jar;/test/nao/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar;/test/nao/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/ignite-spring-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/commons-logging-1.1.1.jar;/test/nao/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar; [2023-07-05T16:03:33,395][INFO ][main][IgniteKernal] >>> +--------------------------------------------------------------------------------+ >>> Ignite ver. 2.16.0-SNAPSHOT#20230704-sha1:a998fe934a0efea0049649a0146179f3cfa61dfd >>> +--------------------------------------------------------------------------------+ >>> OS name: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 >>> CPU(s): -1 >>> Heap: 0.1GB >>> VM name: 19520@devmon-8 >>> Local node [ID=2E17EC7E-AC14-421C-9FEB-5DAE2D251E0A, order=1, clientMode=false] >>> Local node addresses: [localhost/127.0.0.1] >>> Local ports: TCP:10800 TCP:11211 TCP:47900 >>> +--------------------------------------------------------------------------------+ [2023-07-05T16:03:33,396][INFO ][main][IgniteKernal] >>> Ignite cluster is in INACTIVE state (limited functionality available). Use control.(sh|bat) script or IgniteCluster.state(ClusterState.ACTIVE) to change the state. [2023-07-05T16:03:33,404][INFO ][main][GridDiscoveryManager] Topology snapshot [ver=1, locNode=2e17ec7e, servers=1, clients=0, state=INACTIVE, CPUs=-1, offheap=2.4GB, heap=0.1GB, aliveNodes=[IsolatedNode [id=2e17ec7e-ac14-421c-9feb-5dae2d251e0a, consistentId=ise_devmon-8, isClient=false, ver=2.16.0#20230704-sha1:a998fe93]]] [2023-07-05T16:03:33,408][INFO ][main][GridDiscoveryManager] ^-- Baseline [id=0, size=1, online=1, offline=0] [2023-07-05T16:03:33,409][INFO ][main][G] Node started : [stage="Configure system pool" (45 ms),stage="Start managers" (525 ms),stage="Configure binary metadata" (36 ms),stage="Start processors" (596 ms),stage="Init metastore" (346 ms),stage="Init and start regions" (144 ms),stage="Restore binary memory" (321 ms),stage="Restore logical state" (2448 ms),stage="Finish recovery" (2 ms),stage="Join topology" (261 ms),stage="Await transition" (28 ms),stage="Await exchange" (353 ms),stage="Total time" (5105 ms)] [2023-07-05T16:03:39,337][WARN ][defragmentation-worker-#104][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): default [2023-07-05T16:03:44,976][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=88ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=1232ms, pages=2299282, reason='too many dirty pages'] [2023-07-05T16:03:47,601][WARN ][defragmentation-worker-#105][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragPartitionsDataRegion [2023-07-05T16:04:08,264][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2299282, markPos=, walSegmentsCovered=, markDuration=1234ms, pagesWrite=21777ms, fsync=1511ms, total=24610ms] [2023-07-05T16:04:08,951][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=686ms, pages=2299236, reason='partition defragmented'] [2023-07-05T16:04:23,305][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, reason='timeout'] [2023-07-05T16:04:30,659][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2299236, markPos=, walSegmentsCovered=, markDuration=688ms, pagesWrite=20788ms, fsync=919ms, total=22395ms] [2023-07-05T16:04:31,298][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=1ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=638ms, pages=2301191, reason='too many dirty pages'] [2023-07-05T16:04:42,331][WARN ][defragmentation-thread][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragMappingDataRegion [2023-07-05T16:04:54,850][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2301191, markPos=, walSegmentsCovered=, markDuration=639ms, pagesWrite=22108ms, fsync=1444ms, total=24191ms] [2023-07-05T16:04:55,408][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=555ms, pages=2297083, reason='partition defragmented'] [2023-07-05T16:05:18,377][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2297083, markPos=, walSegmentsCovered=, markDuration=558ms, pagesWrite=21532ms, fsync=1437ms, total=23527ms] [2023-07-05T16:05:18,940][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=562ms, pages=2297258, reason='too many dirty pages'] [2023-07-05T16:05:41,793][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2297258, markPos=, walSegmentsCovered=, markDuration=563ms, pagesWrite=21317ms, fsync=1535ms, total=23415ms] [2023-07-05T16:05:42,346][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=551ms, pages=2306190, reason='partition defragmented'] [2023-07-05T16:06:03,708][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=1ms, checkpointLockHoldTime=1ms, reason='timeout'] [2023-07-05T16:06:05,624][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2306190, markPos=, walSegmentsCovered=, markDuration=553ms, pagesWrite=22219ms, fsync=1058ms, total=23830ms] [2023-07-05T16:06:06,147][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=523ms, pages=2287999, reason='too many dirty pages'] [2023-07-05T16:06:31,517][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2287999, markPos=, walSegmentsCovered=, markDuration=523ms, pagesWrite=23648ms, fsync=1722ms, total=25893ms] [2023-07-05T16:06:32,076][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=558ms, pages=2291228, reason='partition defragmented'] [2023-07-05T16:06:56,674][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2291228, markPos=, walSegmentsCovered=, markDuration=559ms, pagesWrite=23110ms, fsync=1487ms, total=25156ms] [2023-07-05T16:06:57,345][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=669ms, pages=2303822, reason='too many dirty pages'] [2023-07-05T16:07:12,894][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=1ms, reason='timeout'] [2023-07-05T16:07:20,125][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2303822, markPos=, walSegmentsCovered=, markDuration=671ms, pagesWrite=21178ms, fsync=1602ms, total=23451ms] [2023-07-05T16:07:20,779][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=654ms, pages=2303748, reason='partition defragmented'] [2023-07-05T16:07:44,941][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2303748, markPos=, walSegmentsCovered=, markDuration=655ms, pagesWrite=22903ms, fsync=1258ms, total=24816ms] [2023-07-05T16:07:45,568][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=626ms, pages=2283281, reason='too many dirty pages'] [2023-07-05T16:08:10,039][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2283281, markPos=, walSegmentsCovered=, markDuration=627ms, pagesWrite=22749ms, fsync=1721ms, total=25097ms] [2023-07-05T16:08:10,704][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=661ms, pages=2298291, reason='partition defragmented'] [2023-07-05T16:08:35,530][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2298291, markPos=, walSegmentsCovered=, markDuration=665ms, pagesWrite=23807ms, fsync=1018ms, total=25490ms] [2023-07-05T16:08:36,223][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=691ms, pages=2284708, reason='too many dirty pages'] [2023-07-05T16:08:58,978][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=2284708, markPos=, walSegmentsCovered=, markDuration=693ms, pagesWrite=20986ms, fsync=1768ms, total=23447ms] [2023-07-05T16:08:59,546][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=566ms, pages=2308763, reason='partition defragmented'] [2023-07-05T16:09:09,169][ERROR][defragmentation-worker-#103][] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=-1368047378, pageIds=[1126441072848775], msg=Runtime failure on row: CacheDataRowAdapter [key=KeyCacheObject [hasValBytes=true], val=CacheObjectByteArrayImpl [arrLen=421], expireTime=0, ver=GridCacheVersion [topVer=299728789, order=0, nodeOrder=1, dataCenterId=0], cacheId=0, link=0101007e0001ef01]]]] org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=-1368047378, pageIds=[1126441072848775], msg=Runtime failure on row: CacheDataRowAdapter [key=KeyCacheObject [hasValBytes=true], val=CacheObjectByteArrayImpl [arrLen=421], expireTime=0, ver=GridCacheVersion [topVer=299728789, order=0, nodeOrder=1, dataCenterId=0], cacheId=0, link=0101007e0001ef01]] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6530) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doPut(BPlusTree.java:2733) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putx(BPlusTree.java:2673) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:751) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] Caused by: java.lang.NullPointerException at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgressImpl.updateEvictedPages(CheckpointProgressImpl.java:253) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1254) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:681) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore$3.allocatePageNoReuse(GridCacheOffheapManager.java:2038) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.allocatePage(DataStructure.java:177) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.allocatePage(DataStructure.java:153) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$2800(BPlusTree.java:215) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Put.insertWithSplit(BPlusTree.java:4042) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Put.insert(BPlusTree.java:4004) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Insert.run0(BPlusTree.java:558) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Insert.run0(BPlusTree.java:539) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:6294) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Insert.run(BPlusTree.java:539) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:6280) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:381) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.write(DataStructure.java:348) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$4800(BPlusTree.java:215) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Put.tryInsert(BPlusTree.java:4162) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3109) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doPut(BPlusTree.java:2693) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] ... 11 more [2023-07-05T16:09:09,183][ERROR][defragmentation-worker-#103][FailureProcessor] A critical problem with persistence data structures was detected. Please make backup of persistence storage and WAL files for further analysis. Persistence storage path: /test/nao/server/work/data WAL path: /test/nao/server/work/wal WAL archive path: /test/nao/server/work/wal_archive [2023-07-05T16:09:09,187][ERROR][defragmentation-worker-#103][FailureProcessor] No deadlocked threads detected. [2023-07-05T16:09:09,711][WARN ][jvm-pause-detector-worker][IgniteKernal] Possible too long JVM pause: 505 milliseconds. [2023-07-05T16:09:09,779][ERROR][defragmentation-worker-#103][FailureProcessor] Thread dump at 2023/07/05 16:09:09 MSK Thread [name="main", id=1, state=WAITING, blockCnt=7, waitCnt=619] Lock [object=java.util.concurrent.CountDownLatch$Sync@2fd1bfd8, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345) at java.base@11.0.17/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232) at app//o.a.i.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:398) Locked synchronizers: java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@312d56ce java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@5f175c2f java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@4fd74c26 Thread [name="Reference Handler", id=2, state=RUNNABLE, blockCnt=3, waitCnt=0] at java.base@11.0.17/java.lang.ref.Reference.waitForReferencePendingList(Native Method) at java.base@11.0.17/java.lang.ref.Reference.processPendingReferences(Reference.java:241) at java.base@11.0.17/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:213) Thread [name="Finalizer", id=3, state=WAITING, blockCnt=4, waitCnt=5] Lock [object=java.lang.ref.ReferenceQueue$Lock@749117ca, ownerName=null, ownerId=-1] at java.base@11.0.17/java.lang.Object.wait(Native Method) at java.base@11.0.17/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.17/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at java.base@11.0.17/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:170) Thread [name="Signal Dispatcher", id=4, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Common-Cleaner", id=19, state=TIMED_WAITING, blockCnt=24, waitCnt=16] Lock [object=java.lang.ref.ReferenceQueue$Lock@1182cb53, ownerName=null, ownerId=-1] at java.base@11.0.17/java.lang.Object.wait(Native Method) at java.base@11.0.17/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@11.0.17/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) at java.base@11.0.17/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:161) Thread [name="Log4j2-TF-6-Scheduled-2", id=23, state=TIMED_WAITING, blockCnt=0, waitCnt=6] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64411339, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.17/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="jvm-pause-detector-worker", id=24, state=TIMED_WAITING, blockCnt=0, waitCnt=6770] at java.base@11.0.17/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:124) Thread [name="ignite-clock", id=25, state=TIMED_WAITING, blockCnt=0, waitCnt=33365] at java.base@11.0.17/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils$25.run(IgniteUtils.java:3646) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-0-#1", id=26, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-1-#2", id=27, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-2-#3", id=28, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-3-#4", id=29, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-4-#5", id=30, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-5-#6", id=31, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-6-#7", id=32, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-7-#8", id=33, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-8-#9", id=34, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-9-#10", id=35, state=WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-10-#11", id=36, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-11-#12", id=37, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-12-#13", id=38, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-13-#14", id=39, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-14-#15", id=40, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-stripe-15-#16", id=41, state=WAITING, blockCnt=0, waitCnt=5] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-0-#17", id=42, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-1-#18", id=43, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-2-#19", id=44, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-3-#20", id=45, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-4-#21", id=46, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-5-#22", id=47, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-6-#23", id=48, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-7-#24", id=49, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-8-#25", id=50, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-9-#26", id=51, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-10-#27", id=52, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-11-#28", id=53, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-12-#29", id=54, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-13-#30", id=55, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-14-#31", id=56, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="data-streamer-stripe-15-#32", id=57, state=WAITING, blockCnt=0, waitCnt=1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:819) at app//o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:623) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-timeout-worker-#38", id=63, state=TIMED_WAITING, blockCnt=3, waitCnt=2030] Lock [object=java.lang.Object@464c5f16, ownerName=null, ownerId=-1] at java.base@11.0.17/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:269) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-comm-#47%TcpCommunicationSpi%", id=72, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@3a9f2f52 - locked sun.nio.ch.EPollSelectorImpl@3dc0a301 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-0-#39%TcpCommunicationSpi%", id=64, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@51f4b75b - locked sun.nio.ch.EPollSelectorImpl@75e4e69c at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-1-#40%TcpCommunicationSpi%", id=65, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@36c2a2cd - locked sun.nio.ch.EPollSelectorImpl@ea2d7bf at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-2-#41%TcpCommunicationSpi%", id=66, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7a18e75f - locked sun.nio.ch.EPollSelectorImpl@281a3e86 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-3-#42%TcpCommunicationSpi%", id=67, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@bed3582 - locked sun.nio.ch.EPollSelectorImpl@136a146f at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-4-#43%TcpCommunicationSpi%", id=68, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2ef52b42 - locked sun.nio.ch.EPollSelectorImpl@5762de14 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-5-#44%TcpCommunicationSpi%", id=69, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@35c1a05f - locked sun.nio.ch.EPollSelectorImpl@764b03e4 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-6-#45%TcpCommunicationSpi%", id=70, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@71e39954 - locked sun.nio.ch.EPollSelectorImpl@30b07b02 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-comm-7-#46%TcpCommunicationSpi%", id=71, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@84bf613 - locked sun.nio.ch.EPollSelectorImpl@263190dd at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="tcp-comm-worker-#1-#48", id=73, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4b759476, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at app//o.a.i.spi.communication.tcp.internal.CommunicationWorker.body(CommunicationWorker.java:167) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at app//o.a.i.spi.communication.tcp.TcpCommunicationSpi$3.body(TcpCommunicationSpi.java:848) at app//o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:58) Thread [name="Thread-0", id=74, state=TIMED_WAITING, blockCnt=14, waitCnt=2309322] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="binary-metadata-writer-#49", id=75, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2f020409, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body0(BinaryMetadataFileStore.java:477) at app//o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body(BinaryMetadataFileStore.java:458) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="page-lock-tracker-timeout", id=76, state=TIMED_WAITING, blockCnt=0, waitCnt=6] at java.base@11.0.17/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.worker.CycleThread.run(CycleThread.java:47) Thread [name="long-qry-#50", id=77, state=TIMED_WAITING, blockCnt=0, waitCnt=340] at java.base@11.0.17/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:8373) at app//o.a.i.i.processors.query.running.HeavyQueriesTracker$1.body(HeavyQueriesTracker.java:104) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-tcp-rest-#64", id=91, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@59fce4c - locked sun.nio.ch.EPollSelectorImpl@622e3cad at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-0-#60", id=87, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6eff6b9f - locked sun.nio.ch.EPollSelectorImpl@51e21805 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-1-#61", id=88, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2af441db - locked sun.nio.ch.EPollSelectorImpl@392cdb7c at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-2-#62", id=89, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2c9aec55 - locked sun.nio.ch.EPollSelectorImpl@6d79fbf3 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-tcp-rest-3-#63", id=90, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@650b851e - locked sun.nio.ch.EPollSelectorImpl@3aa345f8 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-data-loader-flusher-#68", id=95, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@41a0febd, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.DelayQueue.take(DelayQueue.java:217) at app//o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java:106) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="db-checkpoint-thread-defrag-#69", id=96, state=WAITING, blockCnt=16, waitCnt=31] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:142) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.writePages(Checkpointer.java:564) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.doCheckpoint(Checkpointer.java:459) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:278) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-1", id=97, state=TIMED_WAITING, blockCnt=10, waitCnt=2848128] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-2", id=118, state=TIMED_WAITING, blockCnt=18, waitCnt=3046342] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-4", id=119, state=TIMED_WAITING, blockCnt=12, waitCnt=2284952] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-5", id=121, state=TIMED_WAITING, blockCnt=13, waitCnt=2830282] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-3", id=120, state=TIMED_WAITING, blockCnt=14, waitCnt=2598501] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-6", id=122, state=TIMED_WAITING, blockCnt=12, waitCnt=2617445] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-11", id=127, state=RUNNABLE, blockCnt=15, waitCnt=2449271] at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:463) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-10", id=126, state=TIMED_WAITING, blockCnt=16, waitCnt=3129902] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-8", id=125, state=TIMED_WAITING, blockCnt=9, waitCnt=2626309] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-9", id=124, state=RUNNABLE, blockCnt=18, waitCnt=2704544] at java.base@11.0.17/sun.nio.ch.FileDispatcherImpl.pwrite0(Native Method) at java.base@11.0.17/sun.nio.ch.FileDispatcherImpl.pwrite(FileDispatcherImpl.java:68) at java.base@11.0.17/sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:109) at java.base@11.0.17/sun.nio.ch.IOUtil.write(IOUtil.java:58) at java.base@11.0.17/sun.nio.ch.IOUtil.write(IOUtil.java:50) at java.base@11.0.17/sun.nio.ch.SimpleAsynchronousFileChannelImpl$3.run(SimpleAsynchronousFileChannelImpl.java:374) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@2707df6d Thread [name="Thread-7", id=123, state=TIMED_WAITING, blockCnt=10, waitCnt=2537492] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-13", id=129, state=TIMED_WAITING, blockCnt=13, waitCnt=2602670] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-14", id=130, state=RUNNABLE, blockCnt=9, waitCnt=2895965] at java.base@11.0.17/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) at java.base@11.0.17/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) at java.base@11.0.17/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) at java.base@11.0.17/sun.nio.ch.IOUtil.read(IOUtil.java:233) at java.base@11.0.17/sun.nio.ch.IOUtil.read(IOUtil.java:223) at java.base@11.0.17/sun.nio.ch.SimpleAsynchronousFileChannelImpl$2.run(SimpleAsynchronousFileChannelImpl.java:320) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@798348db Thread [name="Thread-15", id=131, state=RUNNABLE, blockCnt=20, waitCnt=2430290] at java.base@11.0.17/jdk.internal.misc.Unsafe.unpark(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:160) at app//o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at app//o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:336) at app//o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:576) at app//o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:555) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO$ChannelOpFuture.completed(AsyncFileIO.java:256) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO$ChannelOpFuture.completed(AsyncFileIO.java:247) at java.base@11.0.17/sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:127) at java.base@11.0.17/sun.nio.ch.SimpleAsynchronousFileChannelImpl$3.run(SimpleAsynchronousFileChannelImpl.java:389) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@79dad505 Thread [name="db-checkpoint-thread-#93", id=135, state=TIMED_WAITING, blockCnt=0, waitCnt=39] Lock [object=o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer@63242319, ownerName=null, ownerId=-1] at java.base@11.0.17/java.lang.Object.wait(Native Method) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.waitCheckpointEvent(Checkpointer.java:820) at app//o.a.i.i.processors.cache.persistence.checkpoint.Checkpointer.body(Checkpointer.java:260) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#94", id=136, state=WAITING, blockCnt=1, waitCnt=67] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5a3e6916, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-cpu-#95", id=137, state=WAITING, blockCnt=0, waitCnt=50] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5a3e6916, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#96", id=138, state=WAITING, blockCnt=4, waitCnt=263] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a927b2c, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#97", id=139, state=WAITING, blockCnt=2, waitCnt=249] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a927b2c, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-thread", id=140, state=WAITING, blockCnt=46, waitCnt=3723908] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.write(AsyncFileIO.java:180) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$5.run(AbstractFileIO.java:117) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.writeFully(AbstractFileIO.java:115) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.write(FilePageStore.java:748) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1252) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$$Lambda$575/0x000000010053fc40.writePage(Unknown Source) at app//o.a.i.i.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:681) at app//o.a.i.i.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore$3.allocatePageNoReuse(GridCacheOffheapManager.java:2038) at app//o.a.i.i.processors.cache.persistence.DataStructure.allocatePage(DataStructure.java:177) at app//o.a.i.i.processors.cache.persistence.DataStructure.allocatePage(DataStructure.java:153) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.access$2800(BPlusTree.java:215) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$Put.insertWithSplit(BPlusTree.java:4042) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$Put.insert(BPlusTree.java:4004) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$Insert.run0(BPlusTree.java:558) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$Insert.run0(BPlusTree.java:539) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:6294) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$Insert.run(BPlusTree.java:539) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:6280) at app//o.a.i.i.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:381) at app//o.a.i.i.processors.cache.persistence.DataStructure.write(DataStructure.java:348) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.access$4800(BPlusTree.java:215) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$Put.tryInsert(BPlusTree.java:4162) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3109) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.doPut(BPlusTree.java:2693) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putx(BPlusTree.java:2673) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:751) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11685) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11587) at app//o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:11567) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.executeDefragmentation(CachePartitionDefragmentationManager.java:379) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction.lambda$execute$0(ExecuteDefragmentationAction.java:73) at app//o.a.i.i.processors.cache.persistence.defragmentation.maintenance.ExecuteDefragmentationAction$$Lambda$961/0x000000010072c440.run(Unknown Source) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="disco-notifier-worker-#98", id=141, state=WAITING, blockCnt=5, waitCnt=4] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3c0127d5, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2830) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2875) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-1", id=142, state=WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7efff7a7, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="disco-event-worker-#99", id=143, state=WAITING, blockCnt=2, waitCnt=3] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@57a047b7, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:3089) at app//o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:3060) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="pool-2-thread-2", id=144, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7efff7a7, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="defragmentation-worker-#100", id=145, state=WAITING, blockCnt=40, waitCnt=3729745] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.write(AsyncFileIO.java:180) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$5.run(AbstractFileIO.java:117) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.writeFully(AbstractFileIO.java:115) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.write(FilePageStore.java:748) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1252) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$$Lambda$575/0x000000010053fc40.writePage(Unknown Source) at app//o.a.i.i.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:681) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.allocateDataPage(AbstractFreeList.java:580) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.writeSinglePage(AbstractFreeList.java:698) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.insertDataRow(AbstractFreeList.java:592) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:74) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:35) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:745) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@559a49da Thread [name="defragmentation-worker-#101", id=146, state=RUNNABLE, blockCnt=37, waitCnt=3738510] at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1010) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324) at java.base@11.0.17/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.releasePage(PageMemoryImpl.java:470) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:329) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@7053d94c Thread [name="defragmentation-worker-#102", id=147, state=RUNNABLE, blockCnt=40, waitCnt=3758424] at java.base@11.0.17/jdk.internal.misc.Unsafe.unpark(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:160) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack$SNode.tryMatch(SynchronousQueue.java:269) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:382) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.offer(SynchronousQueue.java:909) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1347) at java.base@11.0.17/sun.nio.ch.SimpleAsynchronousFileChannelImpl.implRead(SimpleAsynchronousFileChannelImpl.java:339) at java.base@11.0.17/sun.nio.ch.AsynchronousFileChannelImpl.read(AsynchronousFileChannelImpl.java:229) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.read(AsyncFileIO.java:126) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$2.run(AbstractFileIO.java:90) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.readFully(AbstractFileIO.java:88) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.readWithFailover(FilePageStore.java:897) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:512) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:487) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.read(PageReadWriteManagerImpl.java:69) at app//o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:532) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:947) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@aec0099 Thread [name="defragmentation-worker-#103", id=148, state=RUNNABLE, blockCnt=29, waitCnt=3741039] at java.management@11.0.17/sun.management.ThreadImpl.dumpThreads0(Native Method) at java.management@11.0.17/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:521) at java.management@11.0.17/sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:509) at app//o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1534) at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:205) - locked o.a.i.i.processors.failure.FailureProcessor@3e820d74 at app//o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:156) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.processFailure(BPlusTree.java:6545) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6532) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.doPut(BPlusTree.java:2733) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putx(BPlusTree.java:2673) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:751) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@2c86db0f Thread [name="defragmentation-worker-#104", id=149, state=WAITING, blockCnt=39, waitCnt=3735107] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.read(AsyncFileIO.java:129) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$2.run(AbstractFileIO.java:90) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.readFully(AbstractFileIO.java:88) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.readWithFailover(FilePageStore.java:897) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:512) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:487) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.read(PageReadWriteManagerImpl.java:69) at app//o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:532) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:947) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@312f9d62 Thread [name="defragmentation-worker-#105", id=150, state=WAITING, blockCnt=33, waitCnt=3741333] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.read(AsyncFileIO.java:129) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$2.run(AbstractFileIO.java:90) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.readFully(AbstractFileIO.java:88) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.readWithFailover(FilePageStore.java:897) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:512) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:487) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.read(PageReadWriteManagerImpl.java:69) at app//o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:532) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:947) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@664ac68d Thread [name="defragmentation-worker-#106", id=151, state=WAITING, blockCnt=41, waitCnt=3757522] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.read(AsyncFileIO.java:129) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$2.run(AbstractFileIO.java:90) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.readFully(AbstractFileIO.java:88) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.readWithFailover(FilePageStore.java:897) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:512) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:487) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.read(PageReadWriteManagerImpl.java:69) at app//o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:532) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:947) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@1dea10e7 Thread [name="defragmentation-worker-#107", id=152, state=WAITING, blockCnt=44, waitCnt=3741703] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.write(AsyncFileIO.java:180) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$5.run(AbstractFileIO.java:117) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.writeFully(AbstractFileIO.java:115) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.write(FilePageStore.java:748) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1252) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$$Lambda$575/0x000000010053fc40.writePage(Unknown Source) at app//o.a.i.i.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:681) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.allocateDataPage(AbstractFreeList.java:580) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.writeSinglePage(AbstractFreeList.java:698) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.insertDataRow(AbstractFreeList.java:592) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:74) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:35) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:745) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@20025ee4 Thread [name="defragmentation-worker-#108", id=153, state=WAITING, blockCnt=47, waitCnt=3735919] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.read(AsyncFileIO.java:129) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$2.run(AbstractFileIO.java:90) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.readFully(AbstractFileIO.java:88) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.readWithFailover(FilePageStore.java:897) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:512) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:487) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.read(PageReadWriteManagerImpl.java:69) at app//o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:532) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:947) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@27939a8a Thread [name="defragmentation-worker-#109", id=154, state=WAITING, blockCnt=41, waitCnt=3756550] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.write(AsyncFileIO.java:180) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$5.run(AbstractFileIO.java:117) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.writeFully(AbstractFileIO.java:115) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.write(FilePageStore.java:748) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1252) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$$Lambda$575/0x000000010053fc40.writePage(Unknown Source) at app//o.a.i.i.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:681) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.allocateDataPage(AbstractFreeList.java:580) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.writeSinglePage(AbstractFreeList.java:698) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.insertDataRow(AbstractFreeList.java:592) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:74) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:35) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:745) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@618a49b9 Thread [name="defragmentation-worker-#110", id=155, state=RUNNABLE, blockCnt=52, waitCnt=3754132] at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl$Segment.clearRowCache(PageMemoryImpl.java:2314) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl$Segment.tryToRemovePage(PageMemoryImpl.java:2256) at app//o.a.i.i.processors.cache.persistence.pagemem.ClockPageReplacementPolicy.replace(ClockPageReplacementPolicy.java:89) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl$Segment.removePageForReplacement(PageMemoryImpl.java:2421) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:826) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@2e2133aa java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@b88aafe Thread [name="defragmentation-worker-#111", id=156, state=RUNNABLE, blockCnt=38, waitCnt=3753953] at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeLockPage(PageMemoryImpl.java:1691) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeLock(PageMemoryImpl.java:505) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.writeLock(PageMemoryImpl.java:498) at app//o.a.i.i.processors.cache.persistence.tree.util.PageHandler.writeLock(PageHandler.java:437) at app//o.a.i.i.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:365) at app//o.a.i.i.processors.cache.persistence.DataStructure.write(DataStructure.java:348) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.access$4800(BPlusTree.java:215) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree$Put.tryInsert(BPlusTree.java:4162) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3109) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.doPut(BPlusTree.java:2693) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.put(BPlusTree.java:2664) at app//o.a.i.i.processors.cache.persistence.defragmentation.LinkMap.put(LinkMap.java:105) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@20676576 Thread [name="defragmentation-worker-#112", id=157, state=WAITING, blockCnt=47, waitCnt=3746366] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.write(AsyncFileIO.java:180) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$5.run(AbstractFileIO.java:117) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.writeFully(AbstractFileIO.java:115) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.write(FilePageStore.java:748) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1252) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$$Lambda$575/0x000000010053fc40.writePage(Unknown Source) at app//o.a.i.i.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:681) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.allocateDataPage(AbstractFreeList.java:580) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.writeSinglePage(AbstractFreeList.java:698) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.insertDataRow(AbstractFreeList.java:592) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:74) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:35) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:745) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@7fbbaec2 Thread [name="defragmentation-worker-#113", id=158, state=WAITING, blockCnt=36, waitCnt=3746042] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.read(AsyncFileIO.java:129) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$2.run(AbstractFileIO.java:90) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.readFully(AbstractFileIO.java:88) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.readWithFailover(FilePageStore.java:897) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:512) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:487) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.read(PageReadWriteManagerImpl.java:69) at app//o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:532) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:947) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@6efc7dde Thread [name="defragmentation-worker-#114", id=159, state=WAITING, blockCnt=37, waitCnt=3754912] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.read(AsyncFileIO.java:129) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$2.run(AbstractFileIO.java:90) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.readFully(AbstractFileIO.java:88) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.readWithFailover(FilePageStore.java:897) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:512) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:487) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.read(PageReadWriteManagerImpl.java:69) at app//o.a.i.i.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:532) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:947) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:752) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:741) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.doInitFromLink(CacheDataRowAdapter.java:304) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:165) at app//o.a.i.i.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:136) at app//o.a.i.i.processors.cache.tree.DataRow.(DataRow.java:55) at app//o.a.i.i.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:129) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:424) at app//o.a.i.i.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:63) at app//o.a.i.i.processors.cache.persistence.tree.BPlusTree.getRow(BPlusTree.java:5755) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:734) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@17b8774a Thread [name="defragmentation-worker-#115", id=160, state=WAITING, blockCnt=41, waitCnt=3741206] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.write(AsyncFileIO.java:180) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$5.run(AbstractFileIO.java:117) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.writeFully(AbstractFileIO.java:115) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.write(FilePageStore.java:748) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1252) at app//o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$$Lambda$575/0x000000010053fc40.writePage(Unknown Source) at app//o.a.i.i.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:681) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.allocateDataPage(AbstractFreeList.java:580) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.writeSinglePage(AbstractFreeList.java:698) at app//o.a.i.i.processors.cache.persistence.freelist.AbstractFreeList.insertDataRow(AbstractFreeList.java:592) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:74) at app//o.a.i.i.processors.cache.persistence.freelist.CacheFreeList.insertDataRow(CacheFreeList.java:35) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:745) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$1052/0x0000000100765040.apply(Unknown Source) at app//o.a.i.i.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:722) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:569) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) at app//o.a.i.i.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager$$Lambda$979/0x0000000100747440.apply(Unknown Source) at app//o.a.i.i.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11666) at app//o.a.i.i.util.IgniteUtils$$Lambda$990/0x000000010074ac40.call(Unknown Source) at java.base@11.0.17/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@30703a31 Thread [name="pool-2-thread-3", id=161, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7efff7a7, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="exchange-worker-#116", id=162, state=TIMED_WAITING, blockCnt=0, waitCnt=68] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@acb36e7, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:513) at java.base@11.0.17/java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:675) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:3253) at app//o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:3182) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="nio-acceptor-client-listener-#117", id=163, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked sun.nio.ch.Util$2@5f8345f6 - locked sun.nio.ch.EPollSelectorImpl@71c40adf at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3092) at app//o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:3040) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-0-#51", id=78, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3fd59143 - locked sun.nio.ch.EPollSelectorImpl@6c7ae450 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-1-#52", id=79, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2993a0dc - locked sun.nio.ch.EPollSelectorImpl@78d0d5c3 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-2-#53", id=80, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6b1c0c13 - locked sun.nio.ch.EPollSelectorImpl@507f2d44 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-3-#54", id=81, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@64dee2d7 - locked sun.nio.ch.EPollSelectorImpl@1a5e4d42 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-4-#55", id=82, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1129a68e - locked sun.nio.ch.EPollSelectorImpl@3815bce2 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-5-#56", id=83, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2bce296d - locked sun.nio.ch.EPollSelectorImpl@4312450b at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-6-#57", id=84, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1f539631 - locked sun.nio.ch.EPollSelectorImpl@a34db95 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="grid-nio-worker-client-listener-7-#58", id=85, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.base@11.0.17/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.17/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.17/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@50eb1eff - locked sun.nio.ch.EPollSelectorImpl@18033f4 at java.base@11.0.17/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:136) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2272) at app//o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1910) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="services-deployment-worker-#118", id=164, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@261cfeb, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at app//o.a.i.i.processors.service.ServiceDeploymentManager$ServicesDeploymentWorker.body(ServiceDeploymentManager.java:466) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="session-timeout-worker-#59", id=86, state=TIMED_WAITING, blockCnt=0, waitCnt=336] at java.base@11.0.17/java.lang.Thread.sleep(Native Method) at app//o.a.i.i.processors.rest.GridRestProcessor$4.body(GridRestProcessor.java:504) at app//o.a.i.i.util.worker.GridWorker.run(GridWorker.java:125) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="checkpoint-runner-IO-#119", id=176, state=RUNNABLE, blockCnt=0, waitCnt=7852834] at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.checkpoint.LightweightCheckpointManager.lambda$new$0(LightweightCheckpointManager.java:139) at app//o.a.i.i.processors.cache.persistence.checkpoint.LightweightCheckpointManager$$Lambda$785/0x00000001006a3440.write(Unknown Source) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointPagesWriter$1.writePage(CheckpointPagesWriter.java:266) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.copyPageForCheckpoint(PageMemoryImpl.java:1407) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.checkpointWritePage(PageMemoryImpl.java:1296) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointPagesWriter.writePages(CheckpointPagesWriter.java:208) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointPagesWriter.run(CheckpointPagesWriter.java:150) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@50a30cc1 Thread [name="checkpoint-runner-IO-#120", id=177, state=WAITING, blockCnt=0, waitCnt=7876264] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.park(LockSupport.java:323) at app//o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:179) at app//o.a.i.i.util.future.GridFutureAdapter.getUninterruptibly(GridFutureAdapter.java:147) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO.write(AsyncFileIO.java:180) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO$5.run(AbstractFileIO.java:117) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.fully(AbstractFileIO.java:53) at app//o.a.i.i.processors.cache.persistence.file.AbstractFileIO.writeFully(AbstractFileIO.java:115) at app//o.a.i.i.processors.cache.persistence.file.FilePageStore.write(FilePageStore.java:748) at app//o.a.i.i.processors.cache.persistence.pagemem.PageReadWriteManagerImpl.write(PageReadWriteManagerImpl.java:115) at app//o.a.i.i.processors.cache.persistence.checkpoint.LightweightCheckpointManager.lambda$new$0(LightweightCheckpointManager.java:139) at app//o.a.i.i.processors.cache.persistence.checkpoint.LightweightCheckpointManager$$Lambda$785/0x00000001006a3440.write(Unknown Source) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointPagesWriter$1.writePage(CheckpointPagesWriter.java:266) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.copyPageForCheckpoint(PageMemoryImpl.java:1407) at app//o.a.i.i.processors.cache.persistence.pagemem.PageMemoryImpl.checkpointWritePage(PageMemoryImpl.java:1296) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointPagesWriter.writePages(CheckpointPagesWriter.java:208) at app//o.a.i.i.processors.cache.persistence.checkpoint.CheckpointPagesWriter.run(CheckpointPagesWriter.java:150) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@3ea2f9db Thread [name="Thread-18", id=178, state=TIMED_WAITING, blockCnt=12, waitCnt=2369492] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-19", id=179, state=RUNNABLE, blockCnt=16, waitCnt=2373354] at java.base@11.0.17/jdk.internal.misc.Unsafe.unpark(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:160) at app//o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at app//o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:336) at app//o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:576) at app//o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:555) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO$ChannelOpFuture.completed(AsyncFileIO.java:256) at app//o.a.i.i.processors.cache.persistence.file.AsyncFileIO$ChannelOpFuture.completed(AsyncFileIO.java:247) at java.base@11.0.17/sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:127) at java.base@11.0.17/sun.nio.ch.SimpleAsynchronousFileChannelImpl$3.run(SimpleAsynchronousFileChannelImpl.java:389) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@51174e3c Thread [name="Thread-20", id=211, state=TIMED_WAITING, blockCnt=10, waitCnt=1549469] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-21", id=261, state=TIMED_WAITING, blockCnt=3, waitCnt=1028282] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#149", id=309, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="Thread-22", id=310, state=TIMED_WAITING, blockCnt=1, waitCnt=300028] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@782e8e71, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462) at java.base@11.0.17/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361) at java.base@11.0.17/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#150", id=311, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="stat-mgmt-#151", id=312, state=TIMED_WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2631bb9c, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#152", id=313, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#153", id=314, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#154", id=324, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#155", id=325, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#156", id=335, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) Thread [name="sys-#157", id=336, state=TIMED_WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2e255441, ownerName=null, ownerId=-1] at java.base@11.0.17/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.17/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.17/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.17/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:458) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:829) [2023-07-05T16:09:09,829][WARN ][defragmentation-worker-#103][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#100, id=145], state=WAITING Locked pages = [282011847749172[0001007d0001ee34](r=0|w=1)] Locked pages log: name=defragmentation-worker-#100 time=(1688562549796, 2023-07-05 16:09:09.796) L=1 -> Write lock pageId=282011847749172, structureId=cache0-125 [pageIdHex=0001007d0001ee34, partId=125, pageIdx=126516, flags=00000001] Thread=[name=defragmentation-worker-#114, id=159], state=RUNNABLE Locked pages = [1126406713119419[00040076000212bb](r=0|w=1)] Locked pages log: name=defragmentation-worker-#114 time=(1688562549796, 2023-07-05 16:09:09.796) L=1 -> Write lock pageId=1126406713119419, structureId=cache0-p-118##CacheData [pageIdHex=00040076000212bb, partId=118, pageIdx=135867, flags=00000004] [2023-07-05T16:09:09,850][WARN ][defragmentation-worker-#103][DiagnosticProcessor] CorruptedTreeException has occurred. To diagnose it, make a backup of the following directories: [/test/nao/server/work/wal/ise_devmon_8, /test/nao/server/work/wal_archive/ise_devmon_8]. Then, run the following command: java -cp org.apache.ignite.development.utils.IgniteWalConverter walDir=/test/nao/server/work/wal/ise_devmon_8 walArchiveDir=/test/nao/server/work/wal_archive/ise_devmon_8 pageSize=1024 pages=/test/nao/server/work/diagnostic/corruptedPages_2023-07-05_16-09-09_829.txt [2023-07-05T16:09:09,851][ERROR][defragmentation-worker-#103][] JVM will be halted immediately due to the failure: [failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=-1368047378, pageIds=[1126441072848775], msg=Runtime failure on row: CacheDataRowAdapter [key=KeyCacheObject [hasValBytes=true], val=CacheObjectByteArrayImpl [arrLen=421], expireTime=0, ver=GridCacheVersion [topVer=299728789, order=0, nodeOrder=1, dataCenterId=0], cacheId=0, link=0101007e0001ef01]]]]