[2023-07-25T17:55:41,163][WARN ][main][] Property autoActivation deprecated. Use clusterStateOnStart instead. [2023-07-25T17:55:41,282][INFO ][main][IgniteKernal] >>> __________ ________________ >>> / _/ ___/ |/ / _/_ __/ __/ >>> _/ // (7 7 // / / / / _/ >>> /___/\___/_/|_/___/ /_/ /___/ >>> >>> ver. 2.15.0#20230425-sha1:f98f7f35 >>> 2023 Copyright(C) Apache Software Foundation >>> >>> Ignite documentation: https://ignite.apache.org [2023-07-25T17:55:41,286][INFO ][main][IgniteKernal] Config URL: file:/test/steshin/server/config/default-config.xml [2023-07-25T17:55:41,313][INFO ][main][IgniteKernal] IgniteConfiguration [igniteInstanceName=null, pubPoolSize=16, svcPoolSize=16, callbackPoolSize=16, stripedPoolSize=16, sysPoolSize=16, mgmtPoolSize=4, dataStreamerPoolSize=16, utilityCachePoolSize=16, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=16, buildIdxPoolSize=4, igniteHome=/test/steshin/server, igniteWorkDir=/test/steshin/server/work, mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@c667f46, nodeId=eaaa4fa9-9c8b-4d24-9f2b-07dbb1682aac, marsh=BinaryMarshaller [], marshLocJobs=false, p2pEnabled=false, netTimeout=5000, netCompressionLevel=1, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, addressFilter=null, sockTimeout=0, ackTimeout=0, marsh=null, reconCnt=10, reconDelay=2000, maxAckTimeout=600000, soLinger=0, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null, skipAddrsRandomization=false], segPlc=USE_FAILURE_HANDLER, segResolveAttempts=2, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=10000, commSpi=TcpCommunicationSpi [connectGate=org.apache.ignite.spi.communication.tcp.internal.ConnectGateway@7c455e96, ctxInitLatch=java.util.concurrent.CountDownLatch@985696[Count = 1], stopping=false, clientPool=null, nioSrvWrapper=null, stateProvider=null], evtSpi=org.apache.ignite.spi.eventstorage.NoopEventStorageSpi@5bcde458, colSpi=NoopCollisionSpi [], deploySpi=LocalDeploymentSpi [], indexingSpi=org.apache.ignite.spi.indexing.noop.NoopIndexingSpi@675ffd1d, addrRslvr=null, encryptionSpi=org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi@56303475, tracingSpi=org.apache.ignite.spi.tracing.NoopTracingSpi@30506c0d, clientMode=false, rebalanceThreadPoolSize=4, rebalanceTimeout=10000, rebalanceBatchesPrefetchCnt=3, rebalanceThrottle=0, rebalanceBatchSize=524288, txCfg=TransactionConfiguration [txSerEnabled=false, dfltIsolation=REPEATABLE_READ, dfltConcurrency=PESSIMISTIC, dfltTxTimeout=0, txTimeoutOnPartitionMapExchange=0, deadlockTimeout=10000, pessimisticTxLogSize=0, pessimisticTxLogLinger=10000, tmLookupClsName=null, txManagerFactory=null, useJtaSync=false], cacheSanityCheckEnabled=true, discoStartupDelay=60000, deployMode=SHARED, p2pMissedCacheSize=100, locHost=null, timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=30000, sysWorkerBlockedTimeout=null, clientFailureDetectionTimeout=30000, metricsLogFreq=0, connectorCfg=ConnectorConfiguration [jettyPath=null, host=0.0.0.0, port=11211, noDelay=true, directBuf=false, sndBufSize=32768, rcvBufSize=32768, idleQryCurTimeout=600000, idleQryCurCheckFreq=60000, sndQueueLimit=0, selectorCnt=4, idleTimeout=180000, sslEnabled=false, sslClientAuth=false, sslCtxFactory=null, sslFactory=null, portRange=100, threadPoolSize=16, msgInterceptor=null], odbcCfg=null, warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000, cacheMode=PARTITIONED, backups=1, aff=null, grpName=null], classLdr=null, sslCtxFactory=null, platformCfg=null, binaryCfg=null, memCfg=null, pstCfg=null, dsCfg=DataStorageConfiguration [pageSize=1024, concLvl=0, sysDataRegConf=org.apache.ignite.configuration.SystemDataRegionConfiguration@652ab8d9, dfltDataRegConf=DataRegionConfiguration [name=default, maxSize=9663676416, initSize=268435456, swapPath=null, pageEvictionMode=DISABLED, pageReplacementMode=CLOCK, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=true, checkpointPageBufSize=0, lazyMemoryAllocation=true, warmUpCfg=null, memoryAllocator=null, cdcEnabled=false], dataRegions=null, storagePath=/test/steshin/server/work/data, checkpointFreq=60000, lockWaitTime=10000, checkpointThreads=4, checkpointWriteOrder=SEQUENTIAL, walHistSize=20, maxWalArchiveSize=4294967296, walSegments=10, walSegmentSize=67108864, walPath=/test/steshin/server/work/wal, walArchivePath=/test/steshin/server/work/wal_archive, cdcWalPath=db/wal/cdc, cdcWalDirMaxSize=0, metricsEnabled=false, walMode=LOG_ONLY, walTlbSize=131072, walBuffSize=0, walFlushFreq=2000, walFsyncDelay=1000, walRecordIterBuffSize=67108864, alwaysWriteFullPages=false, fileIOFactory=org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory@29852487, metricsSubIntervalCnt=5, metricsRateTimeInterval=60000, walAutoArchiveAfterInactivity=-1, walForceArchiveTimeout=-1, writeThrottlingEnabled=false, walCompactionEnabled=false, walCompactionLevel=1, checkpointReadLockTimeout=null, walPageCompression=DISABLED, walPageCompressionLevel=null, dfltWarmUpCfg=null, encCfg=org.apache.ignite.configuration.EncryptionConfiguration@3afae281, defragmentationThreadPoolSize=28, minWalArchiveSize=-1, memoryAllocator=null], snapshotPath=/test/steshin/backup/snapshot, snapshotThreadPoolSize=10, activeOnStart=true, activeOnStartPropSetFlag=false, autoActivation=false, autoActivationPropSetFlag=true, clusterStateOnStart=null, sqlConnCfg=null, cliConnCfg=ClientConnectorConfiguration [host=0.0.0.0, port=10800, portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true, maxOpenCursorsPerConn=128, threadPoolSize=16, selectorCnt=8, idleTimeout=0, handshakeTimeout=10000, jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false, useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null, thinCliCfg=ThinClientConfiguration [maxActiveTxPerConn=100, maxActiveComputeTasksPerConn=100, sendServerExcStackTraceToClient=false]], mvccVacuumThreadCnt=2, mvccVacuumFreq=5000, authEnabled=false, failureHnd=null, commFailureRslvr=null, sqlCfg=SqlConfiguration [longQryWarnTimeout=3000, dfltQryTimeout=0, sqlQryHistSize=1000, validationEnabled=false], asyncContinuationExecutor=null] [2023-07-25T17:55:41,314][INFO ][main][IgniteKernal] OS: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 [2023-07-25T17:55:41,314][INFO ][main][IgniteKernal] OS user: admin [2023-07-25T17:55:41,319][INFO ][main][IgniteKernal] PID: 10673 [2023-07-25T17:55:41,320][INFO ][main][IgniteKernal] Language runtime: Java Platform API Specification ver. 11 [2023-07-25T17:55:41,321][INFO ][main][IgniteKernal] VM information: OpenJDK Runtime Environment 11.0.18+10-LTS Amazon.com Inc. OpenJDK 64-Bit Server VM 11.0.18+10-LTS [2023-07-25T17:55:41,323][INFO ][main][IgniteKernal] VM total memory: 2.0GB [2023-07-25T17:55:41,324][INFO ][main][IgniteKernal] Remote Management [restart: on, REST: on, JMX (remote: off)] [2023-07-25T17:55:41,324][INFO ][main][IgniteKernal] Logger: Log4J2Logger [quiet=false, config=/test/steshin/server/config/ignite-log4j2.xml] [2023-07-25T17:55:41,328][INFO ][main][IgniteKernal] System cache's DataRegion size is configured to 40 MB. Use DataStorageConfiguration.systemRegionInitialSize property to change the setting. [2023-07-25T17:55:41,329][INFO ][main][IgniteKernal] Configured caches [in 'sysMemPlc' dataRegion: ['ignite-sys-cache']] [2023-07-25T17:55:41,334][INFO ][main][IgniteKernal] 3-rd party licenses can be found at: /test/steshin/server/libs/licenses [2023-07-25T17:55:41,422][INFO ][main][IgnitePluginProcessor] Configured plugins: [2023-07-25T17:55:41,423][INFO ][main][IgnitePluginProcessor] ^-- None [2023-07-25T17:55:41,423][INFO ][main][IgnitePluginProcessor] [2023-07-25T17:55:41,427][INFO ][main][FailureProcessor] Configured failure handler: [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]]] [2023-07-25T17:55:41,707][INFO ][main][TcpCommunicationSpi] Successfully bound communication NIO server to TCP port [port=48020, locHost=/10.40.6.207, selectorsCnt=8, selectorSpins=0, pairedConn=false] [2023-07-25T17:55:41,708][WARN ][main][TcpCommunicationSpi] Message queue limit is set to 0 which may lead to potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and receiver sides. [2023-07-25T17:55:41,741][INFO ][main][GridCollisionManager] Collision resolution is disabled (all jobs will be activated upon arrival). [2023-07-25T17:55:41,786][INFO ][main][MaintenanceProcessor] Resolved store directory for node persistent data: /test/steshin/server/work/data/ise_devmon_8 [2023-07-25T17:55:41,793][INFO ][main][IgniteKernal] Node is being started in maintenance mode. Starting IsolatedDiscoverySpi instead of configured discovery SPI. [2023-07-25T17:55:41,793][INFO ][main][IgniteKernal] Overriding 'clusterStateOnStart' configuration to 'INACTIVE'. [2023-07-25T17:55:41,820][INFO ][main][CacheObjectBinaryProcessorImpl] Resolved directory for serialized binary metadata: /test/steshin/server/work/db/binary_meta/ise_devmon_8 [2023-07-25T17:55:41,858][WARN ][main][GridClusterStateProcessor] Property `autoActivation` will be ignored due to the property `clusterStateOnStart` is presented. [2023-07-25T17:55:41,992][INFO ][main][GridLocalConfigManager] Resolved page store work directory: /test/steshin/server/work/data/ise_devmon_8 [2023-07-25T17:55:42,019][INFO ][main][FilePageStoreManager] Resolved page store work directory: /test/steshin/server/work/data/ise_devmon_8 [2023-07-25T17:55:42,036][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log work directory: /test/steshin/server/work/wal/ise_devmon_8 [2023-07-25T17:55:42,037][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log archive directory: /test/steshin/server/work/wal_archive/ise_devmon_8 [2023-07-25T17:55:42,071][INFO ][main][FileHandleManagerImpl] Initialized write-ahead log manager [mode=LOG_ONLY] [2023-07-25T17:55:42,080][INFO ][main][GridCacheDatabaseSharedManager] Region size was reassigned by defragmentation reason: region = 'default', oldInitialSize = '268435456', newInitialSize = '268435456', oldMaxSize = '9663676416', newMaxSize = '3865470567 [2023-07-25T17:55:42,103][INFO ][main][GridCacheDatabaseSharedManager] Configured data regions initialized successfully [total=7] [2023-07-25T17:55:42,127][INFO ][main][IgniteSnapshotManager] Resolved snapshot work directory: /test/steshin/backup/snapshot [2023-07-25T17:55:42,127][INFO ][main][IgniteSnapshotManager] Resolved temp directory for snapshot creation: /test/steshin/server/work/data/ise_devmon_8/snp [2023-07-25T17:55:42,197][WARN ][main][IgniteH2Indexing] Serialization of Java objects in H2 was enabled. [2023-07-25T17:55:42,411][INFO ][main][ClientListenerProcessor] Client connector processor has started on TCP port 10800 [2023-07-25T17:55:42,474][INFO ][main][GridTcpRestProtocol] Command protocol successfully started [name=TCP binary, host=/0.0.0.0, port=11211] [2023-07-25T17:55:42,533][INFO ][main][IgniteKernal] Non-loopback local IPs: 10.40.6.207, 172.17.0.1 [2023-07-25T17:55:42,533][INFO ][main][IgniteKernal] Enabled local MACs: 0050560AF62B, 02421E6DF0E1 [2023-07-25T17:55:42,541][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/steshin/server/work/data/ise_devmon_8/cp/1688799460403-64bca0f6-16b7-40bb-aa8d-b373b719c49e-START.bin, endMarker=/test/steshin/server/work/data/ise_devmon_8/cp/1688799460403-64bca0f6-16b7-40bb-aa8d-b373b719c49e-END.bin] [2023-07-25T17:55:42,554][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-25T17:55:42,559][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=10966, fileOff=22653145, len=313265], lastMarked=WALPointer [idx=10966, fileOff=22653145, len=313265], lastCheckpointId=64bca0f6-16b7-40bb-aa8d-b373b719c49e] [2023-07-25T17:55:42,633][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=10966, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@20349058, actualFilePtr=WALPointer [idx=10966, fileOff=22966468, len=0]] [2023-07-25T17:55:42,633][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=64bca0f6-16b7-40bb-aa8d-b373b719c49e, pos=WALPointer [idx=10966, fileOff=22653145, len=313265]] [2023-07-25T17:55:42,677][INFO ][main][GridCacheDatabaseSharedManager] Applying lost metastore updates since last checkpoint record [lastMarked=WALPointer [idx=10966, fileOff=22653145, len=313265], lastCheckpointId=64bca0f6-16b7-40bb-aa8d-b373b719c49e] [2023-07-25T17:55:42,700][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=10966, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@2c48cede, actualFilePtr=WALPointer [idx=10966, fileOff=22966468, len=0]] [2023-07-25T17:55:42,706][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=31 ms] [2023-07-25T17:55:42,707][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-25T17:55:42,715][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=0, partitionsProcessed=0, time=0ms] [2023-07-25T17:55:42,781][INFO ][main][GridEncryptionManager] Encryption keys loaded from metastore. [grps=, masterKeyName=null] [2023-07-25T17:55:42,801][INFO ][main][GridClusterStateProcessor] Restoring history for BaselineTopology[id=0] [2023-07-25T17:55:42,831][INFO ][main][ClusterProcessor] Cluster ID and tag has been read from metastorage: ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-25T17:55:42,835][INFO ][main][IgniteClusterImpl] Shutdown policy was updated [oldVal=null, newVal=null] [2023-07-25T17:55:42,837][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustEnabled' was changed from 'null' to 'false' [2023-07-25T17:55:42,838][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustTimeout' was changed from 'null' to '300000' [2023-07-25T17:55:42,838][INFO ][main][IgniteTxManager] Transactions parameter 'txOwnerDumpRequestsAllowed' was changed from 'null' to 'true' [2023-07-25T17:55:42,838][INFO ][main][IgniteTxManager] Transactions parameter 'longOperationsDumpTimeout' was changed from 'null' to '60000' [2023-07-25T17:55:42,839][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpThreshold' was changed from 'null' to '0' [2023-07-25T17:55:42,840][INFO ][main][IgniteTxManager] Transactions parameter 'transactionTimeDumpSamplesCoefficient' was changed from 'null' to '0.0' [2023-07-25T17:55:42,840][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpSamplesPerSecondLimit' was changed from 'null' to '5' [2023-07-25T17:55:42,840][INFO ][main][IgniteTxManager] Transactions parameter 'collisionsDumpInterval' was changed from 'null' to '1000' [2023-07-25T17:55:42,841][INFO ][main][GridCacheDatabaseSharedManager] Historical rebalance WAL threshold changed [property=historical.rebalance.threshold, oldVal=null, newVal=500] [2023-07-25T17:55:42,841][INFO ][main][GridCacheDatabaseSharedManager] Checkpoint frequency deviation changed [oldVal=null, newVal=null] [2023-07-25T17:55:42,843][INFO ][main][IgniteSnapshotManager] The snapshot transfer rate is not limited. [2023-07-25T17:55:42,843][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.disabledFunctions' was changed from 'null' to '[FILE_WRITE, CANCEL_SESSION, MEMORY_USED, CSVREAD, LINK_SCHEMA, MEMORY_FREE, FILE_READ, CSVWRITE, SESSION_ID, LOCK_MODE]' [2023-07-25T17:55:42,843][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.defaultQueryTimeout' was changed from 'null' to '0' [2023-07-25T17:55:42,844][INFO ][main][IgniteStatisticsManagerImpl] Statistics usage state was changed from null to null [2023-07-25T17:55:42,863][INFO ][main][FilePageStoreManager] Cleanup cache stores [total=1, left=0, cleanFiles=false] [2023-07-25T17:55:43,018][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=4.9 GiB, pages=4504080, tableSize=371.4 MiB, replacementSize=550.0 KiB, checkpointBuffer=1.2 GiB] [2023-07-25T17:55:43,022][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-25T17:55:43,024][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-25T17:55:43,028][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-25T17:55:43,047][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=553.0 MiB, pages=500448, tableSize=41.3 MiB, replacementSize=61.1 KiB, checkpointBuffer=256.0 MiB] [2023-07-25T17:55:43,048][INFO ][main][GridCacheDatabaseSharedManager] Data Regions Started: 7 [2023-07-25T17:55:43,049][INFO ][main][IgniteKernal] ^-- defragPartitionsDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=4976MB, maxCfg=4976MB, usedRam=0MB, freeRam=100%, allocRam=4976MB, allocTotal=0MB] ^-- sysMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- default region [type=default, persistence=true, lazyAlloc=true, ... initCfg=256MB, maxCfg=3686MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB] ^-- metastoreMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- TxLog region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- defragMappingDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=552MB, maxCfg=552MB, usedRam=0MB, freeRam=100%, allocRam=552MB, allocTotal=0MB] ^-- volatileDsMemPlc region [type=user, persistence=false, lazyAlloc=true, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=0MB] [2023-07-25T17:55:43,050][INFO ][main][GridCacheDatabaseSharedManager] Starting binary memory restore for: [-2100569601, -593400801] [2023-07-25T17:55:44,081][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/steshin/server/work/data/ise_devmon_8/cp/1688799460403-64bca0f6-16b7-40bb-aa8d-b373b719c49e-START.bin, endMarker=/test/steshin/server/work/data/ise_devmon_8/cp/1688799460403-64bca0f6-16b7-40bb-aa8d-b373b719c49e-END.bin] [2023-07-25T17:55:44,082][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=10966, fileOff=22653145, len=313265], lastMarked=WALPointer [idx=10966, fileOff=22653145, len=313265], lastCheckpointId=64bca0f6-16b7-40bb-aa8d-b373b719c49e] [2023-07-25T17:55:44,130][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=10966, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@5463f035, actualFilePtr=WALPointer [idx=10966, fileOff=22966468, len=0]] [2023-07-25T17:55:44,130][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=64bca0f6-16b7-40bb-aa8d-b373b719c49e, pos=WALPointer [idx=10966, fileOff=22653145, len=313265]] [2023-07-25T17:55:44,134][INFO ][main][GridCacheDatabaseSharedManager] Binary memory state restored at node startup [restoredPtr=WALPointer [idx=10966, fileOff=22966468, len=0]] [2023-07-25T17:55:44,138][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/steshin/server/work/wal/ise_devmon_8/0000000000000006.wal, offset=22966468, ver=2] [2023-07-25T17:55:44,221][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=ignite-sys-cache, id=-2100569601, dataRegionName=sysMemPlc, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647, mvcc=false] [2023-07-25T17:55:44,331][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=3.6 GiB, pages=3336352, tableSize=275.1 MiB, replacementSize=407.4 KiB, checkpointBuffer=921.6 MiB] [2023-07-25T17:55:44,343][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=cache-16384, id=-593400801, dataRegionName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647, mvcc=false] [2023-07-25T17:55:44,346][INFO ][main][GridCacheDatabaseSharedManager] Binary recovery performed in 1295 ms. [2023-07-25T17:55:44,346][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/steshin/server/work/data/ise_devmon_8/cp/1688799460403-64bca0f6-16b7-40bb-aa8d-b373b719c49e-START.bin, endMarker=/test/steshin/server/work/data/ise_devmon_8/cp/1688799460403-64bca0f6-16b7-40bb-aa8d-b373b719c49e-END.bin] [2023-07-25T17:55:44,347][INFO ][main][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=WALPointer [idx=10966, fileOff=22653145, len=313265], lastCheckpointId=64bca0f6-16b7-40bb-aa8d-b373b719c49e] [2023-07-25T17:55:44,370][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=10966, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@3e0a9b1d, actualFilePtr=WALPointer [idx=10966, fileOff=22966497, len=0]] [2023-07-25T17:55:44,377][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=30 ms] [2023-07-25T17:55:44,377][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-25T17:55:49,299][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=2, partitionsProcessed=16484, time=4s] [2023-07-25T17:55:49,302][INFO ][main][MaintenanceProcessor] Node requires maintenance, non-empty set of maintenance tasks is found: [defragmentationMaintenanceTask] [2023-07-25T17:55:49,304][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/steshin/server/work/wal/ise_devmon_8/0000000000000006.wal, offset=22966497, ver=2] [2023-07-25T17:55:49,596][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint started [checkpointId=45d8a285-d20b-49fd-a47c-dbdc73363e2a, startPtr=WALPointer [idx=10966, fileOff=40284385, len=313265], checkpointBeforeLockTime=152ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=44ms, checkpointLockHoldTime=83ms, walCpRecordFsyncDuration=21ms, writeCheckpointEntryDuration=14ms, splitAndSortCpPagesDuration=20ms, pages=16384, reason='node started'] [2023-07-25T17:55:54,803][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint finished [cpId=45d8a285-d20b-49fd-a47c-dbdc73363e2a, pages=16384, markPos=WALPointer [idx=10966, fileOff=40284385, len=313265], walSegmentsCovered=[], markDuration=142ms, pagesWrite=269ms, fsync=4935ms, total=5498ms] [2023-07-25T17:55:54,907][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=44ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=28ms, checkpointLockHoldTime=57ms, reason='beforeDefragmentation'] [2023-07-25T17:55:54,923][INFO ][disco-notifier-worker-#102][MvccProcessorImpl] Assigned mvcc coordinator [crd=MvccCoordinator [topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0], nodeId=eaaa4fa9-9c8b-4d24-9f2b-07dbb1682aac, ver=1690296941795, local=true, initialized=false]] [2023-07-25T17:55:54,927][INFO ][defragmentation-thread][CachePartitionDefragmentationManager] Defragmentation started. [2023-07-25T17:55:54,929][INFO ][sys-#82][ClusterProcessor] Writing cluster ID and tag to metastorage on ready for write ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-25T17:55:54,959][INFO ][defragmentation-thread][CacheGroupContext] Local state for group durability has changed [name=cache-16384, enabled=false] [2023-07-25T17:55:55,214][INFO ][main][IgniteKernal] Security status [authentication=off, sandbox=off, tls/ssl=off] [2023-07-25T17:55:55,215][INFO ][main][IgniteKernal] Performance suggestions for grid (fix if possible) [2023-07-25T17:55:55,215][INFO ][main][IgniteKernal] To disable, set -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true [2023-07-25T17:55:55,215][INFO ][main][IgniteKernal] ^-- Set max direct memory size if getting 'OOME: Direct buffer memory' (add '-XX:MaxDirectMemorySize=[g|G|m|M|k|K]' to JVM options) [2023-07-25T17:55:55,215][INFO ][main][IgniteKernal] ^-- Speed up flushing of dirty pages by OS (alter vm.dirty_expire_centisecs parameter by setting to 500) [2023-07-25T17:55:55,216][INFO ][main][IgniteKernal] ^-- Reduce pages swapping ratio (set vm.swappiness=10 or less) [2023-07-25T17:55:55,216][INFO ][main][IgniteKernal] ^-- Disable assertions (remove '-ea' from JVM options) [2023-07-25T17:55:55,216][INFO ][main][IgniteKernal] Refer to this page for more performance suggestions: https://ignite.apache.org/docs/latest/perf-and-troubleshooting/memory-tuning [2023-07-25T17:55:55,216][INFO ][main][IgniteKernal] [2023-07-25T17:55:55,216][INFO ][main][IgniteKernal] Classpath value: /test/steshin/server/libs/ignite-core-2.15.0.jar:/test/steshin/server/libs/log4j-core-2.20.0.jar:/test/steshin/server/libs/ignite-log4j2-2.15.0.jar:/test/steshin/server/libs/log4j-api-2.20.0.jar:/test/steshin/server/libs/annotations-16.0.3.jar:/test/steshin/server/libs/cache-api-1.0.0.jar:/test/steshin/server/libs/ignite-control-utility/ignite-control-utility-2.15.0.jar:/test/steshin/server/libs/ignite-indexing/h2-1.4.197.jar:/test/steshin/server/libs/ignite-indexing/ignite-indexing-2.15.0.jar:/test/steshin/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar:/test/steshin/server/libs/ignite-indexing/lucene-core-8.11.2.jar:/test/steshin/server/libs/ignite-indexing/commons-codec-1.15.jar:/test/steshin/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar:/test/steshin/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar:/test/steshin/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar:/test/steshin/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar:/test/steshin/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar:/test/steshin/server/libs/ignite-spring/ignite-spring-2.15.0.jar:/test/steshin/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar:/test/steshin/server/libs/ignite-spring/commons-logging-1.1.1.jar:/test/steshin/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar:/test/steshin/server/libs/licenses/* [2023-07-25T17:55:55,217][INFO ][main][IgniteKernal] List of files containing in classpath: /test/steshin/server/libs/ignite-core-2.15.0.jar;/test/steshin/server/libs/log4j-core-2.20.0.jar;/test/steshin/server/libs/ignite-log4j2-2.15.0.jar;/test/steshin/server/libs/log4j-api-2.20.0.jar;/test/steshin/server/libs/annotations-16.0.3.jar;/test/steshin/server/libs/cache-api-1.0.0.jar;/test/steshin/server/libs/ignite-control-utility/ignite-control-utility-2.15.0.jar;/test/steshin/server/libs/ignite-indexing/h2-1.4.197.jar;/test/steshin/server/libs/ignite-indexing/ignite-indexing-2.15.0.jar;/test/steshin/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar;/test/steshin/server/libs/ignite-indexing/lucene-core-8.11.2.jar;/test/steshin/server/libs/ignite-indexing/commons-codec-1.15.jar;/test/steshin/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar;/test/steshin/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar;/test/steshin/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar;/test/steshin/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar;/test/steshin/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar;/test/steshin/server/libs/ignite-spring/ignite-spring-2.15.0.jar;/test/steshin/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar;/test/steshin/server/libs/ignite-spring/commons-logging-1.1.1.jar;/test/steshin/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar; [2023-07-25T17:55:55,380][INFO ][main][IgniteKernal] >>> +-----------------------------------------------------------------------+ >>> Ignite ver. 2.15.0#20230425-sha1:f98f7f35de6dc76a9b69299154afaa2139a5ec6d >>> +-----------------------------------------------------------------------+ >>> OS name: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 >>> CPU(s): -1 >>> Heap: 0.1GB >>> VM name: 10673@devmon-7 >>> Local node [ID=EAAA4FA9-9C8B-4D24-9F2B-07DBB1682AAC, order=1, clientMode=false] >>> Local node addresses: [localhost/127.0.0.1] >>> Local ports: TCP:10800 TCP:11211 TCP:48020 >>> +-----------------------------------------------------------------------+ [2023-07-25T17:55:55,381][INFO ][main][IgniteKernal] >>> Ignite cluster is in INACTIVE state (limited functionality available). Use control.(sh|bat) script or IgniteCluster.state(ClusterState.ACTIVE) to change the state. [2023-07-25T17:55:55,391][INFO ][main][GridDiscoveryManager] Topology snapshot [ver=1, locNode=eaaa4fa9, servers=1, clients=0, state=INACTIVE, CPUs=-1, offheap=3.6GB, heap=0.1GB, aliveNodes=[IsolatedNode [id=eaaa4fa9-9c8b-4d24-9f2b-07dbb1682aac, consistentId=ise_devmon-8, isClient=false, ver=2.15.0#20230425-sha1:f98f7f35]]] [2023-07-25T17:55:55,395][INFO ][main][GridDiscoveryManager] ^-- Baseline [id=0, size=1, online=1, offline=0] [2023-07-25T17:55:55,396][INFO ][main][G] Node started : [stage="Configure system pool" (19 ms),stage="Start managers" (558 ms),stage="Configure binary metadata" (39 ms),stage="Start processors" (677 ms),stage="Init metastore" (333 ms),stage="Init and start regions" (186 ms),stage="Restore binary memory" (1296 ms),stage="Restore logical state" (4953 ms),stage="Finish recovery" (1 ms),stage="Join topology" (5620 ms),stage="Await transition" (13 ms),stage="Await exchange" (459 ms),stage="Total time" (14154 ms)] [2023-07-25T17:55:57,393][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=1ms, checkpointLockWait=36ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=564ms, pages=237920, reason='partition defragmented'] [2023-07-25T17:56:02,489][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=237920, markPos=, walSegmentsCovered=, markDuration=566ms, pagesWrite=4651ms, fsync=444ms, total=5698ms] [2023-07-25T17:56:03,504][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=149ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=864ms, pages=1139012, reason='partition defragmented'] [2023-07-25T17:56:05,409][WARN ][defragmentation-worker-#108][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): default [2023-07-25T17:56:18,877][WARN ][defragmentation-worker-#129][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragPartitionsDataRegion [2023-07-25T17:56:24,278][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=1139012, markPos=, walSegmentsCovered=, markDuration=867ms, pagesWrite=18996ms, fsync=1776ms, total=21788ms] [2023-07-25T17:56:25,709][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=1ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=1430ms, pages=3472458, reason='partition defragmented'] [2023-07-25T17:56:53,420][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=3472458, markPos=, walSegmentsCovered=, markDuration=1432ms, pagesWrite=26910ms, fsync=800ms, total=29142ms] [2023-07-25T17:56:53,522][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, reason='timeout'] [2023-07-25T17:56:55,428][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=2006ms, pages=3461523, reason='partition defragmented'] [2023-07-25T17:57:30,821][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=3461523, markPos=, walSegmentsCovered=, markDuration=2007ms, pagesWrite=29907ms, fsync=5486ms, total=37401ms] [2023-07-25T17:57:32,972][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=1ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=2150ms, pages=3471102, reason='partition defragmented'] [2023-07-25T17:57:38,424][WARN ][defragmentation-worker-#127][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragMappingDataRegion [2023-07-25T17:57:52,722][ERROR][defragmentation-worker-#110][] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=-593400801, pageIds=[1133570718433710], msg=Runtime failure on row: CacheDataRowAdapter [key=KeyCacheObject [hasValBytes=true], val=CacheObjectByteArrayImpl [arrLen=569], expireTime=0, ver=GridCacheVersion [topVer=300147491, order=0, nodeOrder=1, dataCenterId=0], cacheId=0, link=000106fa000001e7]]]] org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=-593400801, pageIds=[1133570718433710], msg=Runtime failure on row: CacheDataRowAdapter [key=KeyCacheObject [hasValBytes=true], val=CacheObjectByteArrayImpl [arrLen=569], expireTime=0, ver=GridCacheVersion [topVer=300147491, order=0, nodeOrder=1, dataCenterId=0], cacheId=0, link=000106fa000001e7]] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:6530) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doPut(BPlusTree.java:2733) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putx(BPlusTree.java:2673) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$copyPartitionData$4(CachePartitionDefragmentationManager.java:747) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.TreeIterator.iterate(TreeIterator.java:83) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.copyPartitionData(CachePartitionDefragmentationManager.java:718) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.defragmentOnePartition(CachePartitionDefragmentationManager.java:565) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.defragmentation.CachePartitionDefragmentationManager.lambda$executeDefragmentation$66155109$1(CachePartitionDefragmentationManager.java:382) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.util.IgniteUtils.lambda$doInParallel$3(IgniteUtils.java:11609) ~[ignite-core-2.15.0.jar:2.15.0] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?] Caused by: java.lang.NullPointerException at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgressImpl.updateEvictedPages(CheckpointProgressImpl.java:253) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.lambda$createPageMemory$6(GridCacheDatabaseSharedManager.java:1254) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.pagemem.DelayedDirtyPageStoreWrite.finishReplacement(DelayedDirtyPageStoreWrite.java:99) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.allocatePage(PageMemoryImpl.java:670) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore$3.allocatePageNoReuse(GridCacheOffheapManager.java:2038) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.allocatePage(DataStructure.java:177) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.allocatePage(DataStructure.java:153) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$2800(BPlusTree.java:215) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Put.insertWithSplit(BPlusTree.java:4042) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Put.insert(BPlusTree.java:4004) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Insert.run0(BPlusTree.java:558) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Insert.run0(BPlusTree.java:539) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:6294) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Insert.run(BPlusTree.java:539) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$GetPageHandler.run(BPlusTree.java:6280) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.writePage(PageHandler.java:381) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.DataStructure.write(DataStructure.java:348) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$4800(BPlusTree.java:215) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Put.tryInsert(BPlusTree.java:4162) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3109) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.putDown(BPlusTree.java:3084) ~[ignite-core-2.15.0.jar:2.15.0] at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.doPut(BPlusTree.java:2693) ~[ignite-core-2.15.0.jar:2.15.0] ... 11 more [2023-07-25T17:57:52,740][ERROR][defragmentation-worker-#110][FailureProcessor] A critical problem with persistence data structures was detected. Please make backup of persistence storage and WAL files for further analysis. Persistence storage path: /test/steshin/server/work/data WAL path: /test/steshin/server/work/wal WAL archive path: /test/steshin/server/work/wal_archive [2023-07-25T17:57:52,787][WARN ][defragmentation-worker-#110][CacheDiagnosticManager] Page locks dump: Thread=[name=defragmentation-worker-#106, id=152], state=RUNNABLE Locked pages = [289081363795308[000106eb00000d6c](r=0|w=1)] Locked pages log: name=defragmentation-worker-#106 time=(1690297072740, 2023-07-25 17:57:52.740) L=1 -> Write lock pageId=289081363795308, structureId=cache-16384-1771 [pageIdHex=000106eb00000d6c, partId=1771, pageIdx=3436, flags=00000001] Thread=[name=defragmentation-worker-#123, id=169], state=RUNNABLE Locked pages = [289193032941666[0001070500000062](r=0|w=1)] Locked pages log: name=defragmentation-worker-#123 time=(1690297072741, 2023-07-25 17:57:52.741) L=1 -> Write lock pageId=289193032941666, structureId=cache-16384-1797 [pageIdHex=0001070500000062, partId=1797, pageIdx=98, flags=00000001] [2023-07-25T17:57:52,796][WARN ][defragmentation-worker-#110][DiagnosticProcessor] CorruptedTreeException has occurred. To diagnose it, make a backup of the following directories: [/test/steshin/server/work/wal/ise_devmon_8, /test/steshin/server/work/wal_archive/ise_devmon_8]. Then, run the following command: java -cp org.apache.ignite.development.utils.IgniteWalConverter walDir=/test/steshin/server/work/wal/ise_devmon_8 walArchiveDir=/test/steshin/server/work/wal_archive/ise_devmon_8 pageSize=1024 pages=/test/steshin/server/work/diagnostic/corruptedPages_2023-07-25_17-57-52_787.txt [2023-07-25T17:57:52,797][ERROR][defragmentation-worker-#110][] JVM will be halted immediately due to the failure: [failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [groupId=-593400801, pageIds=[1133570718433710], msg=Runtime failure on row: CacheDataRowAdapter [key=KeyCacheObject [hasValBytes=true], val=CacheObjectByteArrayImpl [arrLen=569], expireTime=0, ver=GridCacheVersion [topVer=300147491, order=0, nodeOrder=1, dataCenterId=0], cacheId=0, link=000106fa000001e7]]]]