[2023-07-03T17:41:05,018][WARN ][main][] Property autoActivation deprecated. Use clusterStateOnStart instead. [2023-07-03T17:41:05,158][INFO ][main][IgniteKernal] >>> __________ ________________ >>> / _/ ___/ |/ / _/_ __/ __/ >>> _/ // (7 7 // / / / / _/ >>> /___/\___/_/|_/___/ /_/ /___/ >>> >>> ver. 2.16.0-SNAPSHOT#20230619-sha1:3d3862a7 >>> 2023 Copyright(C) Apache Software Foundation >>> >>> Ignite documentation: https://ignite.apache.org [2023-07-03T17:41:05,162][INFO ][main][IgniteKernal] Config URL: file:/test/nao/server/config/default-config.xml [2023-07-03T17:41:05,183][INFO ][main][IgniteKernal] IgniteConfiguration [igniteInstanceName=null, pubPoolSize=16, svcPoolSize=16, callbackPoolSize=16, stripedPoolSize=16, sysPoolSize=16, mgmtPoolSize=4, dataStreamerPoolSize=16, utilityCachePoolSize=16, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=16, buildIdxPoolSize=4, igniteHome=/test/nao/server, igniteWorkDir=/test/nao/server/work, mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@7b50df34, nodeId=a0224bf4-84fc-48ec-bed5-fda556a9c301, marsh=BinaryMarshaller [], marshLocJobs=false, p2pEnabled=true, netTimeout=5000, netCompressionLevel=1, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, addressFilter=null, sockTimeout=0, ackTimeout=0, marsh=null, reconCnt=10, reconDelay=2000, maxAckTimeout=600000, soLinger=0, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null, skipAddrsRandomization=false], segPlc=USE_FAILURE_HANDLER, segResolveAttempts=2, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=10000, commSpi=TcpCommunicationSpi [connectGate=org.apache.ignite.spi.communication.tcp.internal.ConnectGateway@3db64bd4, ctxInitLatch=java.util.concurrent.CountDownLatch@6e106680[Count = 1], stopping=false, clientPool=null, nioSrvWrapper=null, stateProvider=null], evtSpi=org.apache.ignite.spi.eventstorage.NoopEventStorageSpi@576c5536, colSpi=NoopCollisionSpi [], deploySpi=LocalDeploymentSpi [], indexingSpi=org.apache.ignite.spi.indexing.noop.NoopIndexingSpi@7fc645e4, addrRslvr=null, encryptionSpi=org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi@205b132e, tracingSpi=org.apache.ignite.spi.tracing.NoopTracingSpi@73044cdf, clientMode=false, rebalanceThreadPoolSize=4, rebalanceTimeout=10000, rebalanceBatchesPrefetchCnt=3, rebalanceThrottle=0, rebalanceBatchSize=524288, txCfg=TransactionConfiguration [txSerEnabled=false, dfltIsolation=READ_COMMITTED, dfltConcurrency=PESSIMISTIC, dfltTxTimeout=300000, txTimeoutOnPartitionMapExchange=10000, deadlockTimeout=10000, pessimisticTxLogSize=0, pessimisticTxLogLinger=10000, tmLookupClsName=null, txManagerFactory=null, useJtaSync=false], cacheSanityCheckEnabled=true, discoStartupDelay=60000, deployMode=SHARED, p2pMissedCacheSize=100, locHost=null, timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=30000, sysWorkerBlockedTimeout=null, clientFailureDetectionTimeout=30000, metricsLogFreq=0, connectorCfg=ConnectorConfiguration [jettyPath=null, host=0.0.0.0, port=11211, noDelay=true, directBuf=false, sndBufSize=32768, rcvBufSize=32768, idleQryCurTimeout=600000, idleQryCurCheckFreq=60000, sndQueueLimit=0, selectorCnt=4, idleTimeout=180000, sslEnabled=false, sslClientAuth=false, sslFactory=null, portRange=100, threadPoolSize=16, msgInterceptor=null], odbcCfg=null, warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000, cacheMode=PARTITIONED, backups=1, aff=null, grpName=null], classLdr=null, sslCtxFactory=null, platformCfg=null, binaryCfg=null, memCfg=null, pstCfg=null, dsCfg=DataStorageConfiguration [pageSize=1024, concLvl=0, sysDataRegConf=org.apache.ignite.configuration.SystemDataRegionConfiguration@3bb8aabc, dfltDataRegConf=DataRegionConfiguration [name=default, maxSize=2147483648, initSize=268435456, swapPath=null, pageEvictionMode=DISABLED, pageReplacementMode=CLOCK, evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false, metricsSubIntervalCount=5, metricsRateTimeInterval=60000, persistenceEnabled=true, checkpointPageBufSize=0, lazyMemoryAllocation=true, warmUpCfg=null, memoryAllocator=null, cdcEnabled=false], dataRegions=null, storagePath=/test/nao/server/work/data, checkpointFreq=60000, lockWaitTime=10000, checkpointThreads=2, checkpointWriteOrder=SEQUENTIAL, walHistSize=20, maxWalArchiveSize=10737418240, walSegments=10, walSegmentSize=67108864, walPath=/test/nao/server/work/wal, walArchivePath=/test/nao/server/work/wal_archive, cdcWalPath=db/wal/cdc, cdcWalDirMaxSize=0, metricsEnabled=false, walMode=LOG_ONLY, walTlbSize=131072, walBuffSize=0, walFlushFreq=2000, walFsyncDelay=1000, walRecordIterBuffSize=67108864, alwaysWriteFullPages=false, fileIOFactory=org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory@2e3cdec2, metricsSubIntervalCnt=5, metricsRateTimeInterval=60000, walAutoArchiveAfterInactivity=-1, walForceArchiveTimeout=-1, writeThrottlingEnabled=false, walCompactionEnabled=false, walCompactionLevel=1, checkpointReadLockTimeout=null, walPageCompression=DISABLED, walPageCompressionLevel=null, dfltWarmUpCfg=null, encCfg=org.apache.ignite.configuration.EncryptionConfiguration@3051e0b2, defragmentationThreadPoolSize=12, minWalArchiveSize=-1, memoryAllocator=null], snapshotPath=/test/nao/backup/snapshot, snapshotThreadPoolSize=10, activeOnStart=true, activeOnStartPropSetFlag=false, autoActivation=false, autoActivationPropSetFlag=true, clusterStateOnStart=null, sqlConnCfg=null, cliConnCfg=ClientConnectorConfiguration [host=0.0.0.0, port=10800, portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true, maxOpenCursorsPerConn=128, threadPoolSize=16, selectorCnt=8, idleTimeout=0, handshakeTimeout=10000, jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false, useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null, thinCliCfg=ThinClientConfiguration [maxActiveTxPerConn=100, maxActiveComputeTasksPerConn=100, sendServerExcStackTraceToClient=false]], mvccVacuumThreadCnt=2, mvccVacuumFreq=5000, authEnabled=false, failureHnd=null, commFailureRslvr=null, sqlCfg=SqlConfiguration [longQryWarnTimeout=3000, dfltQryTimeout=0, sqlQryHistSize=1000, validationEnabled=false], asyncContinuationExecutor=null] [2023-07-03T17:41:05,184][INFO ][main][IgniteKernal] OS: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 [2023-07-03T17:41:05,184][INFO ][main][IgniteKernal] OS user: admin [2023-07-03T17:41:05,187][INFO ][main][IgniteKernal] PID: 14969 [2023-07-03T17:41:05,188][INFO ][main][IgniteKernal] Language runtime: Java Platform API Specification ver. 11 [2023-07-03T17:41:05,188][INFO ][main][IgniteKernal] VM information: OpenJDK Runtime Environment 11.0.17+8 Eclipse Adoptium OpenJDK 64-Bit Server VM 11.0.17+8 [2023-07-03T17:41:05,190][INFO ][main][IgniteKernal] VM total memory: 2.0GB [2023-07-03T17:41:05,191][INFO ][main][IgniteKernal] Remote Management [restart: on, REST: on, JMX (remote: off)] [2023-07-03T17:41:05,191][INFO ][main][IgniteKernal] Logger: Log4J2Logger [quiet=false, config=/test/nao/server/config/ignite-log4j2.xml] [2023-07-03T17:41:05,193][INFO ][main][IgniteKernal] System cache's DataRegion size is configured to 40 MB. Use DataStorageConfiguration.systemRegionInitialSize property to change the setting. [2023-07-03T17:41:05,194][INFO ][main][IgniteKernal] Configured caches [in 'sysMemPlc' dataRegion: ['ignite-sys-cache']] [2023-07-03T17:41:05,194][WARN ][main][IgniteKernal] Peer class loading is enabled (disable it in production for performance and deployment consistency reasons) [2023-07-03T17:41:05,198][INFO ][main][IgniteKernal] 3-rd party licenses can be found at: /test/nao/server/libs/licenses [2023-07-03T17:41:05,267][INFO ][main][IgnitePluginProcessor] Configured plugins: [2023-07-03T17:41:05,268][INFO ][main][IgnitePluginProcessor] ^-- None [2023-07-03T17:41:05,268][INFO ][main][IgnitePluginProcessor] [2023-07-03T17:41:05,273][INFO ][main][FailureProcessor] Configured failure handler: [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]]] [2023-07-03T17:41:05,587][INFO ][main][TcpCommunicationSpi] Successfully bound communication NIO server to TCP port [port=47900, locHost=/10.40.6.208, selectorsCnt=8, selectorSpins=0, pairedConn=false] [2023-07-03T17:41:05,588][WARN ][main][TcpCommunicationSpi] Message queue limit is set to 0 which may lead to potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and receiver sides. [2023-07-03T17:41:05,622][INFO ][main][GridCollisionManager] Collision resolution is disabled (all jobs will be activated upon arrival). [2023-07-03T17:41:05,673][INFO ][main][MaintenanceProcessor] Resolved store directory for node persistent data: /test/nao/server/work/data/ise_devmon_8 [2023-07-03T17:41:05,679][INFO ][main][IgniteKernal] Node is being started in maintenance mode. Starting IsolatedDiscoverySpi instead of configured discovery SPI. [2023-07-03T17:41:05,680][INFO ][main][IgniteKernal] Overriding 'clusterStateOnStart' configuration to 'INACTIVE'. [2023-07-03T17:41:05,708][INFO ][main][CacheObjectBinaryProcessorImpl] Resolved directory for serialized binary metadata: /test/nao/server/work/db/binary_meta/ise_devmon_8 [2023-07-03T17:41:05,754][WARN ][main][GridClusterStateProcessor] Property `autoActivation` will be ignored due to the property `clusterStateOnStart` is presented. [2023-07-03T17:41:05,910][INFO ][main][GridLocalConfigManager] Resolved page store work directory: /test/nao/server/work/data/ise_devmon_8 [2023-07-03T17:41:05,942][INFO ][main][FilePageStoreManager] Resolved page store work directory: /test/nao/server/work/data/ise_devmon_8 [2023-07-03T17:41:05,944][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log work directory: /test/nao/server/work/wal/ise_devmon_8 [2023-07-03T17:41:05,944][INFO ][main][FileWriteAheadLogManager] Resolved write ahead log archive directory: /test/nao/server/work/wal_archive/ise_devmon_8 [2023-07-03T17:41:05,987][INFO ][main][FileHandleManagerImpl] Initialized write-ahead log manager [mode=LOG_ONLY] [2023-07-03T17:41:05,998][INFO ][main][GridCacheDatabaseSharedManager] Region size was reassigned by defragmentation reason: region = 'default', oldInitialSize = '268435456', newInitialSize = '268435456', oldMaxSize = '2147483648', newMaxSize = '429496730 [2023-07-03T17:41:06,029][INFO ][main][GridCacheDatabaseSharedManager] Configured data regions initialized successfully [total=7] [2023-07-03T17:41:06,058][INFO ][main][IgniteSnapshotManager] Resolved snapshot work directory: /test/nao/backup/snapshot [2023-07-03T17:41:06,058][INFO ][main][IgniteSnapshotManager] Resolved temp directory for snapshot creation: /test/nao/server/work/data/ise_devmon_8/snp [2023-07-03T17:41:06,143][WARN ][main][IgniteH2Indexing] Serialization of Java objects in H2 was enabled. [2023-07-03T17:41:06,329][INFO ][main][ClientListenerProcessor] Client connector processor has started on TCP port 10800 [2023-07-03T17:41:06,389][INFO ][main][GridTcpRestProtocol] Command protocol successfully started [name=TCP binary, host=/0.0.0.0, port=11211] [2023-07-03T17:41:06,454][INFO ][main][IgniteKernal] Non-loopback local IPs: 10.40.6.208, 172.17.0.1 [2023-07-03T17:41:06,455][INFO ][main][IgniteKernal] Enabled local MACs: 0050560AF634, 024206B4ECF3 [2023-07-03T17:41:06,464][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688284431798-4eff1537-9e57-4620-b314-9d62f82ffa2e-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688284431798-4eff1537-9e57-4620-b314-9d62f82ffa2e-END.bin] [2023-07-03T17:41:06,480][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-03T17:41:06,486][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=3223, fileOff=29261474, len=11697], lastMarked=WALPointer [idx=3223, fileOff=29261474, len=11697], lastCheckpointId=4eff1537-9e57-4620-b314-9d62f82ffa2e] [2023-07-03T17:41:06,554][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@120aa40b, actualFilePtr=WALPointer [idx=3223, fileOff=29273200, len=0]] [2023-07-03T17:41:06,554][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=4eff1537-9e57-4620-b314-9d62f82ffa2e, pos=WALPointer [idx=3223, fileOff=29261474, len=11697]] [2023-07-03T17:41:06,607][INFO ][main][GridCacheDatabaseSharedManager] Applying lost metastore updates since last checkpoint record [lastMarked=WALPointer [idx=3223, fileOff=29261474, len=11697], lastCheckpointId=4eff1537-9e57-4620-b314-9d62f82ffa2e] [2023-07-03T17:41:06,628][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@13f40d71, actualFilePtr=WALPointer [idx=3223, fileOff=29273200, len=0]] [2023-07-03T17:41:06,636][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=20 ms] [2023-07-03T17:41:06,637][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-03T17:41:06,647][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=0, partitionsProcessed=0, time=0ms] [2023-07-03T17:41:06,728][INFO ][main][GridEncryptionManager] Encryption keys loaded from metastore. [grps=, masterKeyName=null] [2023-07-03T17:41:06,758][INFO ][main][GridClusterStateProcessor] Restoring history for BaselineTopology[id=0] [2023-07-03T17:41:06,795][INFO ][main][ClusterProcessor] Cluster ID and tag has been read from metastorage: ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-03T17:41:06,800][INFO ][main][IgniteClusterImpl] Shutdown policy was updated [oldVal=null, newVal=null] [2023-07-03T17:41:06,802][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustEnabled' was changed from 'null' to 'false' [2023-07-03T17:41:06,803][INFO ][main][DistributedBaselineConfiguration] Baseline parameter 'baselineAutoAdjustTimeout' was changed from 'null' to '300000' [2023-07-03T17:41:06,803][INFO ][main][IgniteTxManager] Transactions parameter 'txOwnerDumpRequestsAllowed' was changed from 'null' to 'true' [2023-07-03T17:41:06,803][INFO ][main][IgniteTxManager] Transactions parameter 'longOperationsDumpTimeout' was changed from 'null' to '60000' [2023-07-03T17:41:06,804][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpThreshold' was changed from 'null' to '0' [2023-07-03T17:41:06,805][INFO ][main][IgniteTxManager] Transactions parameter 'transactionTimeDumpSamplesCoefficient' was changed from 'null' to '0.0' [2023-07-03T17:41:06,805][INFO ][main][IgniteTxManager] Transactions parameter 'longTransactionTimeDumpSamplesPerSecondLimit' was changed from 'null' to '5' [2023-07-03T17:41:06,806][INFO ][main][IgniteTxManager] Transactions parameter 'collisionsDumpInterval' was changed from 'null' to '1000' [2023-07-03T17:41:06,806][INFO ][main][GridCacheDatabaseSharedManager] Historical rebalance WAL threshold changed [property=historical.rebalance.threshold, oldVal=null, newVal=500] [2023-07-03T17:41:06,807][INFO ][main][GridCacheDatabaseSharedManager] Checkpoint frequency deviation changed [oldVal=null, newVal=null] [2023-07-03T17:41:06,808][INFO ][main][IgniteSnapshotManager] The snapshot transfer rate is not limited. [2023-07-03T17:41:06,808][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.disabledFunctions' was changed from 'null' to '[FILE_WRITE, CANCEL_SESSION, MEMORY_USED, CSVREAD, LINK_SCHEMA, MEMORY_FREE, FILE_READ, CSVWRITE, SESSION_ID, LOCK_MODE]' [2023-07-03T17:41:06,809][INFO ][main][IgniteH2Indexing] SQL parameter 'sql.defaultQueryTimeout' was changed from 'null' to '0' [2023-07-03T17:41:06,809][INFO ][main][IgniteStatisticsManagerImpl] Statistics usage state was changed from null to null [2023-07-03T17:41:06,830][INFO ][main][FilePageStoreManager] Cleanup cache stores [total=1, left=0, cleanFiles=false] [2023-07-03T17:41:06,878][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=1.4 GiB, pages=1334544, tableSize=110.0 MiB, replacementSize=163.0 KiB, checkpointBuffer=368.6 MiB] [2023-07-03T17:41:06,882][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-03T17:41:06,884][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-03T17:41:06,887][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=100.0 MiB, pages=90496, tableSize=7.5 MiB, replacementSize=11.1 KiB, checkpointBuffer=100.0 MiB] [2023-07-03T17:41:06,893][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=163.8 MiB, pages=148272, tableSize=12.2 MiB, replacementSize=18.1 KiB, checkpointBuffer=163.8 MiB] [2023-07-03T17:41:06,894][INFO ][main][GridCacheDatabaseSharedManager] Data Regions Started: 7 [2023-07-03T17:41:06,896][INFO ][main][IgniteKernal] ^-- defragPartitionsDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=1474MB, maxCfg=1474MB, usedRam=0MB, freeRam=100%, allocRam=1474MB, allocTotal=0MB] ^-- sysMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- default region [type=default, persistence=true, lazyAlloc=true, ... initCfg=256MB, maxCfg=409MB, usedRam=0MB, freeRam=100%, allocRam=0MB, allocTotal=0MB] ^-- metastoreMemPlc region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- TxLog region [type=internal, persistence=true, lazyAlloc=false, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=100MB, allocTotal=0MB] ^-- defragMappingDataRegion region [type=user, persistence=true, lazyAlloc=false, ... initCfg=163MB, maxCfg=163MB, usedRam=0MB, freeRam=100%, allocRam=163MB, allocTotal=0MB] ^-- volatileDsMemPlc region [type=user, persistence=false, lazyAlloc=true, ... initCfg=40MB, maxCfg=100MB, usedRam=0MB, freeRam=100%, allocRam=0MB] [2023-07-03T17:41:06,896][INFO ][main][GridCacheDatabaseSharedManager] Starting binary memory restore for: [-2100569601, -1368047378] [2023-07-03T17:41:06,982][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688284431798-4eff1537-9e57-4620-b314-9d62f82ffa2e-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688284431798-4eff1537-9e57-4620-b314-9d62f82ffa2e-END.bin] [2023-07-03T17:41:06,982][INFO ][main][GridCacheDatabaseSharedManager] Checking memory state [lastValidPos=WALPointer [idx=3223, fileOff=29261474, len=11697], lastMarked=WALPointer [idx=3223, fileOff=29261474, len=11697], lastCheckpointId=4eff1537-9e57-4620-b314-9d62f82ffa2e] [2023-07-03T17:41:07,027][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@72b2c5ed, actualFilePtr=WALPointer [idx=3223, fileOff=29273200, len=0]] [2023-07-03T17:41:07,028][INFO ][main][GridCacheDatabaseSharedManager] Found last checkpoint marker [cpId=4eff1537-9e57-4620-b314-9d62f82ffa2e, pos=WALPointer [idx=3223, fileOff=29261474, len=11697]] [2023-07-03T17:41:07,033][INFO ][main][GridCacheDatabaseSharedManager] Binary memory state restored at node startup [restoredPtr=WALPointer [idx=3223, fileOff=29273200, len=0]] [2023-07-03T17:41:07,041][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/nao/server/work/wal/ise_devmon_8/0000000000000003.wal, offset=29273200, ver=2] [2023-07-03T17:41:07,157][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=ignite-sys-cache, id=-2100569601, dataRegionName=sysMemPlc, mode=REPLICATED, atomicity=TRANSACTIONAL, backups=2147483647, mvcc=false] [2023-07-03T17:41:07,174][INFO ][main][PageMemoryImpl] Started page memory [memoryAllocated=409.6 MiB, pages=370704, tableSize=30.6 MiB, replacementSize=45.4 KiB, checkpointBuffer=256.0 MiB] [2023-07-03T17:41:07,209][INFO ][main][GridCacheProcessor] Started cache in recovery mode [name=cache0, id=-1368047378, dataRegionName=default, mode=REPLICATED, atomicity=ATOMIC, backups=2147483647, mvcc=false] [2023-07-03T17:41:07,212][INFO ][main][GridCacheDatabaseSharedManager] Binary recovery performed in 316 ms. [2023-07-03T17:41:07,213][INFO ][main][CheckpointMarkersStorage] Read checkpoint status [startMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688284431798-4eff1537-9e57-4620-b314-9d62f82ffa2e-START.bin, endMarker=/test/nao/server/work/data/ise_devmon_8/cp/1688284431798-4eff1537-9e57-4620-b314-9d62f82ffa2e-END.bin] [2023-07-03T17:41:07,214][INFO ][main][GridCacheDatabaseSharedManager] Applying lost cache updates since last checkpoint record [lastMarked=WALPointer [idx=3223, fileOff=29261474, len=11697], lastCheckpointId=4eff1537-9e57-4620-b314-9d62f82ffa2e] [2023-07-03T17:41:07,238][WARN ][main][FileWriteAheadLogManager] WAL segment tail reached. [idx=3223, isWorkDir=true, serVer=org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer@73aaec54, actualFilePtr=WALPointer [idx=3223, fileOff=29273229, len=0]] [2023-07-03T17:41:07,242][INFO ][main][GridCacheDatabaseSharedManager] Finished applying WAL changes [updatesApplied=0, time=30 ms] [2023-07-03T17:41:07,242][INFO ][main][GridCacheProcessor] Restoring partition state for local groups. [2023-07-03T17:41:08,537][WARN ][sys-#89][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): default [2023-07-03T17:41:10,230][INFO ][main][GridCacheProcessor] Finished restoring partition state for local groups [groupsProcessed=2, partitionsProcessed=612, time=2s] [2023-07-03T17:41:10,234][INFO ][main][MaintenanceProcessor] Node requires maintenance, non-empty set of maintenance tasks is found: [defragmentationMaintenanceTask] [2023-07-03T17:41:10,237][INFO ][main][FileWriteAheadLogManager] Resuming logging to WAL segment [file=/test/nao/server/work/wal/ise_devmon_8/0000000000000003.wal, offset=29273229, ver=2] [2023-07-03T17:41:10,302][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint started [checkpointId=3065d370-6b3e-4d50-9494-0f262ec1f2a4, startPtr=WALPointer [idx=3223, fileOff=29814413, len=11697], checkpointBeforeLockTime=31ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=4ms, checkpointLockHoldTime=10ms, walCpRecordFsyncDuration=4ms, writeCheckpointEntryDuration=11ms, splitAndSortCpPagesDuration=4ms, pages=512, reason='node started'] [2023-07-03T17:41:10,504][INFO ][db-checkpoint-thread-#93][Checkpointer] Checkpoint finished [cpId=3065d370-6b3e-4d50-9494-0f262ec1f2a4, pages=512, markPos=WALPointer [idx=3223, fileOff=29814413, len=11697], walSegmentsCovered=[], markDuration=31ms, pagesWrite=33ms, fsync=167ms, total=262ms] [2023-07-03T17:41:10,515][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=4ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=3ms, checkpointLockHoldTime=5ms, reason='beforeDefragmentation'] [2023-07-03T17:41:10,520][INFO ][defragmentation-thread][CachePartitionDefragmentationManager] Defragmentation started. [2023-07-03T17:41:10,527][INFO ][defragmentation-thread][CacheGroupContext] Local state for group durability has changed [name=cache0, enabled=false] [2023-07-03T17:41:10,532][INFO ][disco-notifier-worker-#98][MvccProcessorImpl] Assigned mvcc coordinator [crd=MvccCoordinator [topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0], nodeId=a0224bf4-84fc-48ec-bed5-fda556a9c301, ver=1688395265681, local=true, initialized=false]] [2023-07-03T17:41:10,538][INFO ][sys-#77][ClusterProcessor] Writing cluster ID and tag to metastorage on ready for write ClusterIdAndTag [id=8be4a483-4613-4368-af96-99a5169fb106, tag=angry_lovelace] [2023-07-03T17:41:10,798][INFO ][main][IgniteKernal] Security status [authentication=off, sandbox=off, tls/ssl=off] [2023-07-03T17:41:10,799][INFO ][main][IgniteKernal] Performance suggestions for grid (fix if possible) [2023-07-03T17:41:10,799][INFO ][main][IgniteKernal] To disable, set -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true [2023-07-03T17:41:10,799][INFO ][main][IgniteKernal] ^-- Set max direct memory size if getting 'OOME: Direct buffer memory' (add '-XX:MaxDirectMemorySize=[g|G|m|M|k|K]' to JVM options) [2023-07-03T17:41:10,799][INFO ][main][IgniteKernal] ^-- Speed up flushing of dirty pages by OS (alter vm.dirty_expire_centisecs parameter by setting to 500) [2023-07-03T17:41:10,800][INFO ][main][IgniteKernal] ^-- Reduce pages swapping ratio (set vm.swappiness=10 or less) [2023-07-03T17:41:10,800][INFO ][main][IgniteKernal] ^-- Disable assertions (remove '-ea' from JVM options) [2023-07-03T17:41:10,800][INFO ][main][IgniteKernal] Refer to this page for more performance suggestions: https://ignite.apache.org/docs/latest/perf-and-troubleshooting/memory-tuning [2023-07-03T17:41:10,800][INFO ][main][IgniteKernal] [2023-07-03T17:41:10,800][INFO ][main][IgniteKernal] Classpath value: /test/nao/server/libs/log4j-core-2.20.0.jar:/test/nao/server/libs/log4j-api-2.20.0.jar:/test/nao/server/libs/ignite-log4j2-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-core-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/annotations-16.0.3.jar:/test/nao/server/libs/cache-api-1.0.0.jar:/test/nao/server/libs/ignite-control-utility/ignite-control-utility-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-indexing/h2-1.4.197.jar:/test/nao/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar:/test/nao/server/libs/ignite-indexing/ignite-indexing-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-indexing/lucene-core-8.11.2.jar:/test/nao/server/libs/ignite-indexing/commons-codec-1.15.jar:/test/nao/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar:/test/nao/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/ignite-spring-2.16.0-SNAPSHOT.jar:/test/nao/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar:/test/nao/server/libs/ignite-spring/commons-logging-1.1.1.jar:/test/nao/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar:/test/nao/server/libs/licenses/* [2023-07-03T17:41:10,801][INFO ][main][IgniteKernal] List of files containing in classpath: /test/nao/server/libs/log4j-core-2.20.0.jar;/test/nao/server/libs/log4j-api-2.20.0.jar;/test/nao/server/libs/ignite-log4j2-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-core-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/annotations-16.0.3.jar;/test/nao/server/libs/cache-api-1.0.0.jar;/test/nao/server/libs/ignite-control-utility/ignite-control-utility-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-indexing/h2-1.4.197.jar;/test/nao/server/libs/ignite-indexing/lucene-analyzers-common-8.11.2.jar;/test/nao/server/libs/ignite-indexing/ignite-indexing-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-indexing/lucene-core-8.11.2.jar;/test/nao/server/libs/ignite-indexing/commons-codec-1.15.jar;/test/nao/server/libs/ignite-indexing/lucene-queryparser-8.11.2.jar;/test/nao/server/libs/ignite-spring/spring-beans-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-core-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/ignite-spring-2.16.0-SNAPSHOT.jar;/test/nao/server/libs/ignite-spring/spring-context-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-jcl-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/spring-expression-5.2.22.RELEASE.jar;/test/nao/server/libs/ignite-spring/commons-logging-1.1.1.jar;/test/nao/server/libs/ignite-spring/spring-aop-5.2.22.RELEASE.jar; [2023-07-03T17:41:10,834][INFO ][main][IgniteKernal] >>> +--------------------------------------------------------------------------------+ >>> Ignite ver. 2.16.0-SNAPSHOT#20230619-sha1:3d3862a7272476fd008fd5f2d9eee2a33db164eb >>> +--------------------------------------------------------------------------------+ >>> OS name: Linux 3.10.0-1160.53.1.el7.x86_64 amd64 >>> CPU(s): -1 >>> Heap: 0.1GB >>> VM name: 14969@devmon-8 >>> Local node [ID=A0224BF4-84FC-48EC-BED5-FDA556A9C301, order=1, clientMode=false] >>> Local node addresses: [localhost/127.0.0.1] >>> Local ports: TCP:10800 TCP:11211 TCP:47900 >>> +--------------------------------------------------------------------------------+ [2023-07-03T17:41:10,834][INFO ][main][IgniteKernal] >>> Ignite cluster is in INACTIVE state (limited functionality available). Use control.(sh|bat) script or IgniteCluster.state(ClusterState.ACTIVE) to change the state. [2023-07-03T17:41:10,845][INFO ][main][GridDiscoveryManager] Topology snapshot [ver=1, locNode=a0224bf4, servers=1, clients=0, state=INACTIVE, CPUs=-1, offheap=0.4GB, heap=0.1GB, aliveNodes=[IsolatedNode [id=a0224bf4-84fc-48ec-bed5-fda556a9c301, consistentId=ise_devmon-8, isClient=false, ver=2.16.0#20230619-sha1:3d3862a7]]] [2023-07-03T17:41:10,849][INFO ][main][GridDiscoveryManager] ^-- Baseline [id=0, size=1, online=1, offline=0] [2023-07-03T17:41:10,850][INFO ][main][G] Node started : [stage="Configure system pool" (21 ms),stage="Start managers" (573 ms),stage="Configure binary metadata" (46 ms),stage="Start processors" (704 ms),stage="Init metastore" (379 ms),stage="Init and start regions" (65 ms),stage="Restore binary memory" (316 ms),stage="Restore logical state" (3018 ms),stage="Finish recovery" (2 ms),stage="Join topology" (297 ms),stage="Await transition" (21 ms),stage="Await exchange" (297 ms),stage="Total time" (5739 ms)] [2023-07-03T17:41:17,643][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=16ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=770ms, pages=1037413, reason='too many dirty pages'] [2023-07-03T17:41:19,012][WARN ][defragmentation-thread][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragPartitionsDataRegion [2023-07-03T17:41:29,452][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=1037413, markPos=, walSegmentsCovered=, markDuration=772ms, pagesWrite=11161ms, fsync=646ms, total=12595ms] [2023-07-03T17:41:29,760][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=306ms, pages=1030790, reason='too many dirty pages'] [2023-07-03T17:41:43,456][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=1030790, markPos=, walSegmentsCovered=, markDuration=308ms, pagesWrite=12869ms, fsync=827ms, total=14004ms] [2023-07-03T17:41:43,729][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=270ms, pages=1035191, reason='too many dirty pages'] [2023-07-03T17:41:49,587][WARN ][defragmentation-worker-#101][PageMemoryImpl] Page replacements started, pages will be rotated with disk, this will affect storage performance (consider increasing DataRegionConfiguration#setMaxSize for data region): defragMappingDataRegion [2023-07-03T17:41:57,192][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=1035191, markPos=, walSegmentsCovered=, markDuration=274ms, pagesWrite=12482ms, fsync=980ms, total=13736ms] [2023-07-03T17:41:57,447][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=1ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=253ms, pages=1033689, reason='partition defragmented'] [2023-07-03T17:42:10,985][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=1033689, markPos=, walSegmentsCovered=, markDuration=255ms, pagesWrite=12754ms, fsync=784ms, total=13793ms] [2023-07-03T17:42:11,433][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=0ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=446ms, pages=1036194, reason='too many dirty pages'] [2023-07-03T17:42:12,697][INFO ][db-checkpoint-thread-#93][Checkpointer] Skipping checkpoint (no pages were modified) [checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, reason='timeout'] [2023-07-03T17:42:22,656][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint finished [cpId=, pages=1036194, markPos=, walSegmentsCovered=, markDuration=449ms, pagesWrite=10957ms, fsync=265ms, total=11671ms] [2023-07-03T17:42:22,928][INFO ][db-checkpoint-thread-defrag-#69][Checkpointer] Checkpoint started [checkpointId=, startPtr=, checkpointBeforeLockTime=1ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=0ms, checkpointLockHoldTime=0ms, walCpRecordFsyncDuration=0ms, writeCheckpointEntryDuration=0ms, splitAndSortCpPagesDuration=270ms, pages=1041556, reason='too many dirty pages'] [2023-07-03T17:42:43,936][ERROR][db-checkpoint-thread-defrag-#69][] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.IgniteCheckedException: Compound exception for CountDownFuture.]] org.apache.ignite.IgniteCheckedException: Compound exception for CountDownFuture. at org.apache.ignite.internal.util.future.CountDownFuture.addError(CountDownFuture.java:72) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.future.CountDownFuture.onDone(CountDownFuture.java:46) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.future.CountDownFuture.onDone(CountDownFuture.java:28) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:543) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointPagesWriter.run(CheckpointPagesWriter.java:167) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:829) [?:?] Suppressed: java.lang.AssertionError: Invalid state. Type is 0! pageId = 0001000d00024cbf at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.copyPageForCheckpoint(PageMemoryImpl.java:1359) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.checkpointWritePage(PageMemoryImpl.java:1277) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointPagesWriter.writePages(CheckpointPagesWriter.java:208) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointPagesWriter.run(CheckpointPagesWriter.java:150) ~[ignite-core-2.16.0-SNAPSHOT.jar:2.16.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:829) [?:?] [2023-07-03T17:42:43,954][WARN ][db-checkpoint-thread-defrag-#69][CacheDiagnosticManager] Page locks dump: [2023-07-03T17:42:43,955][ERROR][db-checkpoint-thread-defrag-#69][] JVM will be halted immediately due to the failure: [failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.IgniteCheckedException: Compound exception for CountDownFuture.]]