2020-04-20 01:29:04.572 [db-checkpoint-thread-#73%DataCenter%] INFO o.a.i.i.p.c.p.GridCacheDatabaseSharedManager - Checkpoint started [checkpointId=fcab5ad0-dfd2-4fe5-b6df-91e32bd36136, startPtr=FileWALPointer [idx=14, fileOff=60683710, len=40871], checkpointBeforeLockTime=34ms, checkpointLockWait=0ms, checkpointListenersExecuteTime=12ms, checkpointLockHoldTime=16ms, walCpRecordFsyncDuration=3ms, writeCheckpointEntryDuration=2ms, splitAndSortCpPagesDuration=6ms, pages=8066, reason='timeout'] 2020-04-20 01:29:04.673 [pub-#475%DataCenter%] ERROR o.a.i.i.p.q.h.t.GridMapQueryExecutor - Failed to execute local query. org.apache.ignite.internal.processors.query.IgniteSQLException: General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199] at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQuery(IgniteH2Indexing.java:794) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQueryWithTimer(IgniteH2Indexing.java:882) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest0(GridMapQueryExecutor.java:411) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest(GridMapQueryExecutor.java:243) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.onMessage(IgniteH2Indexing.java:2183) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:149) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:144) at org.apache.ignite.internal.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:37) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.send(IgniteH2Indexing.java:2308) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.send(GridReduceQueryExecutor.java:1133) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.query(GridReduceQueryExecutor.java:455) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing$7.iterator(IgniteH2Indexing.java:1701) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iter(QueryCursorImpl.java:101) at org.apache.ignite.internal.processors.cache.query.RegisteredQueryCursor.iter(RegisteredQueryCursor.java:69) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iterator(QueryCursorImpl.java:91) at com.sinitek.data.api.persistence.CacheQueryKt.firstRow(CacheQuery.kt:339) at com.sinitek.data.api.persistence.CacheQueryKt.firstLongValue(CacheQuery.kt:344) at com.sinitek.data.sync.persistence.DataSourceVersion$Companion.incrementCount(DataSourceVersion.kt:49) at com.sinitek.data.sync.service.SyncServiceImpl.incrementDataSourceUpdated(SyncServiceImpl.kt:478) at sun.reflect.GeneratedMethodAccessor71.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyCallable.call(GridServiceProxy.java:425) at org.apache.ignite.internal.processors.closure.GridClosureProcessor$C2.execute(GridClosureProcessor.java:1854) at org.apache.ignite.internal.processors.job.GridJobWorker$2.call(GridJobWorker.java:567) at org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:7080) at org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:561) at org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:490) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.h2.jdbc.JdbcSQLNonTransientException: General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199] at org.h2.message.DbException.getJdbcSQLException(DbException.java:503) at org.h2.message.DbException.getJdbcSQLException(DbException.java:427) at org.h2.message.DbException.get(DbException.java:194) at org.h2.message.DbException.convert(DbException.java:347) at org.h2.command.Command.executeQuery(Command.java:219) at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:115) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQuery(IgniteH2Indexing.java:784) ... 31 common frames omitted Caused by: java.lang.ArrayIndexOutOfBoundsException: 32768 at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getStore(FilePageStoreManager.java:1176) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:516) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:503) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:872) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:698) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:687) at org.apache.ignite.internal.processors.cache.persistence.DataStructure.acquirePage(DataStructure.java:156) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.acquirePage(BPlusTree.java:6041) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5595) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.next(BPlusTree.java:5839) at org.apache.ignite.internal.processors.query.h2.H2Cursor.next(H2Cursor.java:63) at org.h2.index.IndexCursor.next(IndexCursor.java:258) at org.h2.table.TableFilter.next(TableFilter.java:513) at org.h2.command.dml.Select$LazyResultQueryFlat.fetchNextRow(Select.java:1930) at org.h2.result.LazyResult.hasNext(LazyResult.java:101) at org.h2.result.LazyResult.next(LazyResult.java:60) at org.h2.command.dml.Select.queryFlat(Select.java:750) at org.h2.command.dml.Select.queryWithoutCache(Select.java:903) at org.h2.command.dml.Query.queryWithoutCacheLazyCheck(Query.java:151) at org.h2.command.dml.Query.query(Query.java:415) at org.h2.command.dml.Query.query(Query.java:397) at org.h2.command.CommandContainer.query(CommandContainer.java:145) at org.h2.command.Command.executeQuery(Command.java:202) ... 33 common frames omitted 2020-04-20 01:29:04.674 [pub-#475%DataCenter%] ERROR o.a.i.i.p.q.h.t.GridMapQueryExecutor - Failed to run map query on local node. org.apache.ignite.internal.processors.query.IgniteSQLException: General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199] at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQuery(IgniteH2Indexing.java:794) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQueryWithTimer(IgniteH2Indexing.java:882) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest0(GridMapQueryExecutor.java:411) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest(GridMapQueryExecutor.java:243) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.onMessage(IgniteH2Indexing.java:2183) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:149) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:144) at org.apache.ignite.internal.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:37) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.send(IgniteH2Indexing.java:2308) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.send(GridReduceQueryExecutor.java:1133) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.query(GridReduceQueryExecutor.java:455) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing$7.iterator(IgniteH2Indexing.java:1701) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iter(QueryCursorImpl.java:101) at org.apache.ignite.internal.processors.cache.query.RegisteredQueryCursor.iter(RegisteredQueryCursor.java:69) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iterator(QueryCursorImpl.java:91) at com.sinitek.data.api.persistence.CacheQueryKt.firstRow(CacheQuery.kt:339) at com.sinitek.data.api.persistence.CacheQueryKt.firstLongValue(CacheQuery.kt:344) at com.sinitek.data.sync.persistence.DataSourceVersion$Companion.incrementCount(DataSourceVersion.kt:49) at com.sinitek.data.sync.service.SyncServiceImpl.incrementDataSourceUpdated(SyncServiceImpl.kt:478) at sun.reflect.GeneratedMethodAccessor71.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyCallable.call(GridServiceProxy.java:425) at org.apache.ignite.internal.processors.closure.GridClosureProcessor$C2.execute(GridClosureProcessor.java:1854) at org.apache.ignite.internal.processors.job.GridJobWorker$2.call(GridJobWorker.java:567) at org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:7080) at org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:561) at org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:490) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.h2.jdbc.JdbcSQLNonTransientException: General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199] at org.h2.message.DbException.getJdbcSQLException(DbException.java:503) at org.h2.message.DbException.getJdbcSQLException(DbException.java:427) at org.h2.message.DbException.get(DbException.java:194) at org.h2.message.DbException.convert(DbException.java:347) at org.h2.command.Command.executeQuery(Command.java:219) at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:115) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQuery(IgniteH2Indexing.java:784) ... 31 common frames omitted Caused by: java.lang.ArrayIndexOutOfBoundsException: 32768 at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getStore(FilePageStoreManager.java:1176) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:516) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:503) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:872) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:698) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:687) at org.apache.ignite.internal.processors.cache.persistence.DataStructure.acquirePage(DataStructure.java:156) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.acquirePage(BPlusTree.java:6041) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5595) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.next(BPlusTree.java:5839) at org.apache.ignite.internal.processors.query.h2.H2Cursor.next(H2Cursor.java:63) at org.h2.index.IndexCursor.next(IndexCursor.java:258) at org.h2.table.TableFilter.next(TableFilter.java:513) at org.h2.command.dml.Select$LazyResultQueryFlat.fetchNextRow(Select.java:1930) at org.h2.result.LazyResult.hasNext(LazyResult.java:101) at org.h2.result.LazyResult.next(LazyResult.java:60) at org.h2.command.dml.Select.queryFlat(Select.java:750) at org.h2.command.dml.Select.queryWithoutCache(Select.java:903) at org.h2.command.dml.Query.queryWithoutCacheLazyCheck(Query.java:151) at org.h2.command.dml.Query.query(Query.java:415) at org.h2.command.dml.Query.query(Query.java:397) at org.h2.command.CommandContainer.query(CommandContainer.java:145) at org.h2.command.Command.executeQuery(Command.java:202) ... 33 common frames omitted 2020-04-20 01:29:04.678 [pub-#475%DataCenter%] ERROR o.a.i.i.processors.job.GridJobWorker - Failed to execute job [jobId=f1c8d239171-db868bb1-a49a-4fe6-a0b3-80846d54d8f6, ses=GridJobSessionImpl [ses=GridTaskSessionImpl [taskName=o.a.i.i.processors.service.GridServiceProxy$ServiceProxyCallable, dep=GridDeployment [ts=1587311402893, depMode=SHARED, clsLdr=sun.misc.Launcher$AppClassLoader@18b4aac2, clsLdrId=734d1239171-b3b73915-4c02-40c1-a434-7e46389c6292, userVer=0, loc=true, sampleClsName=o.a.i.i.processors.cluster.ClusterProcessor, pendingUndeploy=false, undeployed=false, usage=2], taskClsName=o.a.i.i.processors.service.GridServiceProxy$ServiceProxyCallable, sesId=d1c8d239171-db868bb1-a49a-4fe6-a0b3-80846d54d8f6, startTime=1587317344621, endTime=9223372036854775807, taskNodeId=db868bb1-a49a-4fe6-a0b3-80846d54d8f6, clsLdr=sun.misc.Launcher$AppClassLoader@18b4aac2, closed=false, cpSpi=null, failSpi=null, loadSpi=null, usage=1, fullSup=false, internal=false, topPred=null, subjId=db868bb1-a49a-4fe6-a0b3-80846d54d8f6, mapFut=IgniteFuture [orig=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=314226523]], execName=null], jobId=f1c8d239171-db868bb1-a49a-4fe6-a0b3-80846d54d8f6]] org.apache.ignite.IgniteException: javax.cache.CacheException: Failed to execute map query on remote node [nodeId=b3b73915-4c02-40c1-a434-7e46389c6292, errMsg=General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199]] at org.apache.ignite.internal.processors.closure.GridClosureProcessor$C2.execute(GridClosureProcessor.java:1857) at org.apache.ignite.internal.processors.job.GridJobWorker$2.call(GridJobWorker.java:567) at org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:7080) at org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:561) at org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:490) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyException: javax.cache.CacheException: Failed to execute map query on remote node [nodeId=b3b73915-4c02-40c1-a434-7e46389c6292, errMsg=General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199]] at org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyCallable.call(GridServiceProxy.java:428) at org.apache.ignite.internal.processors.closure.GridClosureProcessor$C2.execute(GridClosureProcessor.java:1854) ... 8 common frames omitted Caused by: javax.cache.CacheException: Failed to execute map query on remote node [nodeId=b3b73915-4c02-40c1-a434-7e46389c6292, errMsg=General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199]] at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.fail(GridReduceQueryExecutor.java:234) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.onFail(GridReduceQueryExecutor.java:209) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.sendError(GridMapQueryExecutor.java:682) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest0(GridMapQueryExecutor.java:516) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest(GridMapQueryExecutor.java:243) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.onMessage(IgniteH2Indexing.java:2183) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:149) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:144) at org.apache.ignite.internal.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:37) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.send(IgniteH2Indexing.java:2308) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.send(GridReduceQueryExecutor.java:1133) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.query(GridReduceQueryExecutor.java:455) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing$7.iterator(IgniteH2Indexing.java:1701) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iter(QueryCursorImpl.java:101) at org.apache.ignite.internal.processors.cache.query.RegisteredQueryCursor.iter(RegisteredQueryCursor.java:69) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iterator(QueryCursorImpl.java:91) at com.sinitek.data.api.persistence.CacheQueryKt.firstRow(CacheQuery.kt:339) at com.sinitek.data.api.persistence.CacheQueryKt.firstLongValue(CacheQuery.kt:344) at com.sinitek.data.sync.persistence.DataSourceVersion$Companion.incrementCount(DataSourceVersion.kt:49) at com.sinitek.data.sync.service.SyncServiceImpl.incrementDataSourceUpdated(SyncServiceImpl.kt:478) at sun.reflect.GeneratedMethodAccessor71.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyCallable.call(GridServiceProxy.java:425) ... 9 common frames omitted Caused by: org.apache.ignite.internal.processors.query.IgniteSQLMapStepException: Failed to execute map query on remote node [nodeId=b3b73915-4c02-40c1-a434-7e46389c6292, errMsg=General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199]] at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.fail(GridReduceQueryExecutor.java:230) ... 32 common frames omitted Caused by: org.apache.ignite.internal.processors.query.IgniteSQLException: General error: "java.lang.ArrayIndexOutOfBoundsException: 32768"; SQL statement: SELECT "__Z0"."COUNT" AS "__C0_0" FROM "DATACENTER"."DATASOURCEVERSION" AS "__Z0" WHERE ("__Z0"."OWNER" = ?4) AND (("__Z0"."VERSION" = ?3) AND (("__Z0"."DATASOURCEID" = ?1) AND ("__Z0"."POSITION" = ?2))) FETCH FIRST ROW ONLY [50000-199] ... 33 common frames omitted 2020-04-20 01:29:06.286 [pub-#481%DataCenter%] ERROR ROOT - Critical system error detected. Will be handled accordingly to configured handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [pages(groupId, pageId)=[IgniteBiTuple [val1=-1496539969, val2=140737488355328]], cacheId=-107682605, cacheName=DATASOURCEVERSION, indexName=DATA_SOURCE_VERSION_QUERY_IDX, groupName=DataCenter, msg=Runtime failure on cursor iteration]]] org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [pages(groupId, pageId)=[IgniteBiTuple [val1=-1496539969, val2=140737488355328]], cacheId=-107682605, cacheName=DATASOURCEVERSION, indexName=DATA_SOURCE_VERSION_QUERY_IDX, groupName=DataCenter, msg=Runtime failure on cursor iteration] at org.apache.ignite.internal.processors.query.h2.database.H2Tree.corruptedTreeException(H2Tree.java:849) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5616) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.next(BPlusTree.java:5839) at org.apache.ignite.internal.processors.query.h2.H2Cursor.next(H2Cursor.java:63) at org.h2.index.IndexCursor.next(IndexCursor.java:258) at org.h2.table.TableFilter.next(TableFilter.java:513) at org.h2.command.dml.Select$LazyResultQueryFlat.fetchNextRow(Select.java:1930) at org.h2.result.LazyResult.hasNext(LazyResult.java:101) at org.h2.result.LazyResult.next(LazyResult.java:60) at org.h2.command.dml.Select.queryFlat(Select.java:750) at org.h2.command.dml.Select.queryWithoutCache(Select.java:903) at org.h2.command.dml.Query.queryWithoutCacheLazyCheck(Query.java:151) at org.h2.command.dml.Query.query(Query.java:415) at org.h2.command.dml.Query.query(Query.java:397) at org.h2.command.CommandContainer.query(CommandContainer.java:145) at org.h2.command.Command.executeQuery(Command.java:202) at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:115) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQuery(IgniteH2Indexing.java:784) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeSqlQueryWithTimer(IgniteH2Indexing.java:882) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest0(GridMapQueryExecutor.java:411) at org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest(GridMapQueryExecutor.java:243) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.onMessage(IgniteH2Indexing.java:2183) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:149) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:144) at org.apache.ignite.internal.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:37) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.send(IgniteH2Indexing.java:2308) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.send(GridReduceQueryExecutor.java:1133) at org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.query(GridReduceQueryExecutor.java:455) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing$7.iterator(IgniteH2Indexing.java:1701) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iter(QueryCursorImpl.java:101) at org.apache.ignite.internal.processors.cache.QueryCursorImpl.iterator(QueryCursorImpl.java:91) at org.apache.ignite.internal.processors.query.h2.dml.DmlUtils.doUpdate(DmlUtils.java:245) at org.apache.ignite.internal.processors.query.h2.dml.DmlUtils.processSelectResult(DmlUtils.java:168) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeUpdateNonTransactional(IgniteH2Indexing.java:2930) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeUpdate(IgniteH2Indexing.java:2783) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeUpdateDistributed(IgniteH2Indexing.java:2713) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.executeDml(IgniteH2Indexing.java:1170) at org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.querySqlFields(IgniteH2Indexing.java:1096) at org.apache.ignite.internal.processors.query.GridQueryProcessor$4.applyx(GridQueryProcessor.java:2377) at org.apache.ignite.internal.processors.query.GridQueryProcessor$4.applyx(GridQueryProcessor.java:2373) at org.apache.ignite.internal.util.lang.IgniteOutClosureX.apply(IgniteOutClosureX.java:35) at org.apache.ignite.internal.processors.query.GridQueryProcessor.executeQuery(GridQueryProcessor.java:2889) at org.apache.ignite.internal.processors.query.GridQueryProcessor.lambda$querySqlFields$1(GridQueryProcessor.java:2393) at org.apache.ignite.internal.processors.query.GridQueryProcessor.executeQuerySafe(GridQueryProcessor.java:2431) at org.apache.ignite.internal.processors.query.GridQueryProcessor.querySqlFields(GridQueryProcessor.java:2367) at org.apache.ignite.internal.processors.query.GridQueryProcessor.querySqlFields(GridQueryProcessor.java:2294) at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.query(IgniteCacheProxyImpl.java:799) at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.query(IgniteCacheProxyImpl.java:747) at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.query(GatewayProtectedCacheProxy.java:385) at com.sinitek.data.api.persistence.CacheQuery$DefaultImpls.query(CacheQuery.kt:100) at com.sinitek.data.api.persistence.CacheQueryWithId.query(CacheQuery.kt:336) at com.sinitek.data.api.persistence.CacheQuery$DefaultImpls.sqlFieldsQuery(CacheQuery.kt:198) at com.sinitek.data.api.persistence.CacheQueryWithId.sqlFieldsQuery(CacheQuery.kt:336) at com.sinitek.data.api.persistence.CacheQuery$DefaultImpls.sqlFieldsQuery$default(CacheQuery.kt:176) at com.sinitek.data.sync.persistence.DataSourceVersion$Companion.incrementCount(DataSourceVersion.kt:55) at com.sinitek.data.sync.service.SyncServiceImpl.incrementDataSourceUpdated(SyncServiceImpl.kt:478) at sun.reflect.GeneratedMethodAccessor71.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyCallable.call(GridServiceProxy.java:425) at org.apache.ignite.internal.processors.closure.GridClosureProcessor$C2.execute(GridClosureProcessor.java:1854) at org.apache.ignite.internal.processors.job.GridJobWorker$2.call(GridJobWorker.java:567) at org.apache.ignite.internal.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:7080) at org.apache.ignite.internal.processors.job.GridJobWorker.execute0(GridJobWorker.java:561) at org.apache.ignite.internal.processors.job.GridJobWorker.body(GridJobWorker.java:490) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.IllegalStateException: Unknown page type: 0 pageId: 0000000000000000 at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.io(BPlusTree.java:5331) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.access$100(BPlusTree.java:98) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5604) ... 67 common frames omitted 2020-04-20 01:29:06.286 [pub-#481%DataCenter%] ERROR o.a.i.i.p.failure.FailureProcessor - A critical problem with persistence data structures was detected. Please make backup of persistence storage and WAL files for further analysis. Persistence storage path: null WAL path: db/wal WAL archive path: db/wal/archive 2020-04-20 01:29:06.288 [pub-#481%DataCenter%] ERROR o.a.i.i.p.failure.FailureProcessor - No deadlocked threads detected. 2020-04-20 01:29:06.609 [pub-#481%DataCenter%] ERROR o.a.i.i.p.failure.FailureProcessor - Thread dump at 2020/04/20 01:29:06 CST Thread [name="query-#502%DataCenter%", id=987, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@52eb2b20, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="query-#501%DataCenter%", id=986, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@52eb2b20, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="query-#500%DataCenter%", id=985, state=TIMED_WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@52eb2b20, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-220", id=983, state=TIMED_WAITING, blockCnt=0, waitCnt=918] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@1ea32662, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-219", id=982, state=TIMED_WAITING, blockCnt=0, waitCnt=1631] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@1ea32662, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-221", id=984, state=TIMED_WAITING, blockCnt=0, waitCnt=1609] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@1ea32662, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-209", id=966, state=TIMED_WAITING, blockCnt=0, waitCnt=6672] Lock [object=java.util.concurrent.SynchronousQueue$TransferStack@1ea32662, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#496%DataCenter%", id=768, state=TIMED_WAITING, blockCnt=2, waitCnt=24527] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#495%DataCenter%", id=767, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#494%DataCenter%", id=766, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#493%DataCenter%", id=765, state=TIMED_WAITING, blockCnt=0, waitCnt=23868] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#492%DataCenter%", id=764, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#491%DataCenter%", id=763, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#490%DataCenter%", id=762, state=TIMED_WAITING, blockCnt=0, waitCnt=23661] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#489%DataCenter%", id=761, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#488%DataCenter%", id=760, state=TIMED_WAITING, blockCnt=0, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#487%DataCenter%", id=759, state=TIMED_WAITING, blockCnt=2, waitCnt=23574] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#486%DataCenter%", id=758, state=TIMED_WAITING, blockCnt=0, waitCnt=7992] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#485%DataCenter%", id=757, state=TIMED_WAITING, blockCnt=3, waitCnt=7995] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#484%DataCenter%", id=756, state=TIMED_WAITING, blockCnt=1021, waitCnt=23557] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#483%DataCenter%", id=755, state=TIMED_WAITING, blockCnt=0, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#482%DataCenter%", id=754, state=TIMED_WAITING, blockCnt=4, waitCnt=7995] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#481%DataCenter%", id=753, state=RUNNABLE, blockCnt=0, waitCnt=23857] at sun.management.ThreadImpl.dumpThreads0(Native Method) at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:454) at o.a.i.i.util.IgniteUtils.dumpThreads(IgniteUtils.java:1457) at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:187) - locked o.a.i.i.processors.failure.FailureProcessor@2c955cad at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:150) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.processFailure(BPlusTree.java:6125) at o.a.i.i.processors.query.h2.database.H2Tree.corruptedTreeException(H2Tree.java:851) at o.a.i.i.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5616) at o.a.i.i.processors.cache.persistence.tree.BPlusTree$ForwardCursor.next(BPlusTree.java:5839) at o.a.i.i.processors.query.h2.H2Cursor.next(H2Cursor.java:63) at org.h2.index.IndexCursor.next(IndexCursor.java:258) at org.h2.table.TableFilter.next(TableFilter.java:513) at org.h2.command.dml.Select$LazyResultQueryFlat.fetchNextRow(Select.java:1930) at org.h2.result.LazyResult.hasNext(LazyResult.java:101) at org.h2.result.LazyResult.next(LazyResult.java:60) at org.h2.command.dml.Select.queryFlat(Select.java:750) at org.h2.command.dml.Select.queryWithoutCache(Select.java:903) at org.h2.command.dml.Query.queryWithoutCacheLazyCheck(Query.java:151) at org.h2.command.dml.Query.query(Query.java:415) at org.h2.command.dml.Query.query(Query.java:397) at org.h2.command.CommandContainer.query(CommandContainer.java:145) at org.h2.command.Command.executeQuery(Command.java:202) - locked org.h2.engine.Session@2d5975f6 at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:115) - locked org.h2.engine.Session@2d5975f6 at o.a.i.i.processors.query.h2.IgniteH2Indexing.executeSqlQuery(IgniteH2Indexing.java:784) at o.a.i.i.processors.query.h2.IgniteH2Indexing.executeSqlQueryWithTimer(IgniteH2Indexing.java:882) at o.a.i.i.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest0(GridMapQueryExecutor.java:411) at o.a.i.i.processors.query.h2.twostep.GridMapQueryExecutor.onQueryRequest(GridMapQueryExecutor.java:243) at o.a.i.i.processors.query.h2.IgniteH2Indexing.onMessage(IgniteH2Indexing.java:2183) at o.a.i.i.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:149) at o.a.i.i.processors.query.h2.twostep.GridReduceQueryExecutor$1.applyx(GridReduceQueryExecutor.java:144) at o.a.i.i.util.lang.IgniteInClosure2X.apply(IgniteInClosure2X.java:37) at o.a.i.i.processors.query.h2.IgniteH2Indexing.send(IgniteH2Indexing.java:2308) at o.a.i.i.processors.query.h2.twostep.GridReduceQueryExecutor.send(GridReduceQueryExecutor.java:1133) at o.a.i.i.processors.query.h2.twostep.GridReduceQueryExecutor.query(GridReduceQueryExecutor.java:455) at o.a.i.i.processors.query.h2.IgniteH2Indexing$7.iterator(IgniteH2Indexing.java:1701) at o.a.i.i.processors.cache.QueryCursorImpl.iter(QueryCursorImpl.java:101) at o.a.i.i.processors.cache.QueryCursorImpl.iterator(QueryCursorImpl.java:91) at o.a.i.i.processors.query.h2.dml.DmlUtils.doUpdate(DmlUtils.java:245) at o.a.i.i.processors.query.h2.dml.DmlUtils.processSelectResult(DmlUtils.java:168) at o.a.i.i.processors.query.h2.IgniteH2Indexing.executeUpdateNonTransactional(IgniteH2Indexing.java:2930) at o.a.i.i.processors.query.h2.IgniteH2Indexing.executeUpdate(IgniteH2Indexing.java:2783) at o.a.i.i.processors.query.h2.IgniteH2Indexing.executeUpdateDistributed(IgniteH2Indexing.java:2713) at o.a.i.i.processors.query.h2.IgniteH2Indexing.executeDml(IgniteH2Indexing.java:1170) at o.a.i.i.processors.query.h2.IgniteH2Indexing.querySqlFields(IgniteH2Indexing.java:1096) at o.a.i.i.processors.query.GridQueryProcessor$4.applyx(GridQueryProcessor.java:2377) at o.a.i.i.processors.query.GridQueryProcessor$4.applyx(GridQueryProcessor.java:2373) at o.a.i.i.util.lang.IgniteOutClosureX.apply(IgniteOutClosureX.java:35) at o.a.i.i.processors.query.GridQueryProcessor.executeQuery(GridQueryProcessor.java:2889) at o.a.i.i.processors.query.GridQueryProcessor.lambda$querySqlFields$1(GridQueryProcessor.java:2393) at o.a.i.i.processors.query.GridQueryProcessor$$Lambda$584/29695983.get(Unknown Source) at o.a.i.i.processors.query.GridQueryProcessor.executeQuerySafe(GridQueryProcessor.java:2431) at o.a.i.i.processors.query.GridQueryProcessor.querySqlFields(GridQueryProcessor.java:2367) at o.a.i.i.processors.query.GridQueryProcessor.querySqlFields(GridQueryProcessor.java:2294) at o.a.i.i.processors.cache.IgniteCacheProxyImpl.query(IgniteCacheProxyImpl.java:799) at o.a.i.i.processors.cache.IgniteCacheProxyImpl.query(IgniteCacheProxyImpl.java:747) at o.a.i.i.processors.cache.GatewayProtectedCacheProxy.query(GatewayProtectedCacheProxy.java:385) at com.sinitek.data.api.persistence.CacheQuery$DefaultImpls.query(CacheQuery.kt:100) at com.sinitek.data.api.persistence.CacheQueryWithId.query(CacheQuery.kt:336) at com.sinitek.data.api.persistence.CacheQuery$DefaultImpls.sqlFieldsQuery(CacheQuery.kt:198) at com.sinitek.data.api.persistence.CacheQueryWithId.sqlFieldsQuery(CacheQuery.kt:336) at com.sinitek.data.api.persistence.CacheQuery$DefaultImpls.sqlFieldsQuery$default(CacheQuery.kt:176) at com.sinitek.data.sync.persistence.DataSourceVersion$Companion.incrementCount(DataSourceVersion.kt:55) at com.sinitek.data.sync.service.SyncServiceImpl.incrementDataSourceUpdated(SyncServiceImpl.kt:478) at sun.reflect.GeneratedMethodAccessor71.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at o.a.i.i.processors.service.GridServiceProxy$ServiceProxyCallable.call(GridServiceProxy.java:425) at o.a.i.i.processors.closure.GridClosureProcessor$C2.execute(GridClosureProcessor.java:1854) at o.a.i.i.processors.job.GridJobWorker$2.call(GridJobWorker.java:567) at o.a.i.i.util.IgniteUtils.wrapThreadLoader(IgniteUtils.java:7080) at o.a.i.i.processors.job.GridJobWorker.execute0(GridJobWorker.java:561) at o.a.i.i.processors.job.GridJobWorker.body(GridJobWorker.java:490) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.locks.ReentrantLock$NonfairSync@568d1592 java.util.concurrent.ThreadPoolExecutor$Worker@3ed39ff Thread [name="svc-#480%DataCenter%", id=752, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#479%DataCenter%", id=751, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#478%DataCenter%", id=750, state=TIMED_WAITING, blockCnt=2, waitCnt=23995] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#477%DataCenter%", id=749, state=TIMED_WAITING, blockCnt=0, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#476%DataCenter%", id=748, state=TIMED_WAITING, blockCnt=11, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#475%DataCenter%", id=747, state=TIMED_WAITING, blockCnt=3, waitCnt=23607] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#474%DataCenter%", id=746, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#473%DataCenter%", id=745, state=TIMED_WAITING, blockCnt=0, waitCnt=7995] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#472%DataCenter%", id=744, state=TIMED_WAITING, blockCnt=135, waitCnt=23710] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#471%DataCenter%", id=743, state=TIMED_WAITING, blockCnt=0, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#470%DataCenter%", id=742, state=TIMED_WAITING, blockCnt=158, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pool-8-thread-1", id=741, state=WAITING, blockCnt=12, waitCnt=37254] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:177) at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:140) at o.a.i.i.processors.service.GridServiceProxy.invokeMethod(GridServiceProxy.java:191) at o.a.i.i.processors.service.GridServiceProxy$ProxyInvocationHandler.invoke(GridServiceProxy.java:363) at com.sun.proxy.$Proxy62.exists(Unknown Source) at com.sinitek.data.sync.service.SyncServiceImpl.putTableDeletedItems(SyncServiceImpl.kt:250) at com.sinitek.data.sync.service.SyncServiceImpl.putDeletedItems(SyncServiceImpl.kt:194) at com.sinitek.data.sync.service.SyncServiceImpl.sync(SyncServiceImpl.kt:149) at com.sinitek.data.sync.service.SyncServiceImpl$execute$$inlined$forEach$lambda$1$1.run(SyncServiceImpl.kt:78) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5e6b6b2c o.a.i.i.processors.datastructures.GridCacheLockImpl$Sync@6acfc67d Thread [name="pub-#467%DataCenter%", id=736, state=TIMED_WAITING, blockCnt=0, waitCnt=23663] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#466%DataCenter%", id=735, state=TIMED_WAITING, blockCnt=0, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#465%DataCenter%", id=734, state=TIMED_WAITING, blockCnt=0, waitCnt=7993] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="mgmt-#464%DataCenter%", id=733, state=TIMED_WAITING, blockCnt=0, waitCnt=23984] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6cb99fbd, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#463%DataCenter%", id=732, state=TIMED_WAITING, blockCnt=1, waitCnt=23832] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#462%DataCenter%", id=731, state=TIMED_WAITING, blockCnt=0, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#461%DataCenter%", id=730, state=TIMED_WAITING, blockCnt=15, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="mgmt-#460%DataCenter%", id=729, state=TIMED_WAITING, blockCnt=0, waitCnt=23983] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6cb99fbd, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="pub-#459%DataCenter%", id=728, state=TIMED_WAITING, blockCnt=0, waitCnt=23650] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c0b1b8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="svc-#458%DataCenter%", id=727, state=TIMED_WAITING, blockCnt=0, waitCnt=7994] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@203b6285, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="utility-#457%DataCenter%", id=726, state=TIMED_WAITING, blockCnt=101, waitCnt=7995] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b82ff89, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="mgmt-#456%DataCenter%", id=725, state=TIMED_WAITING, blockCnt=0, waitCnt=23982] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6cb99fbd, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="mgmt-#454%DataCenter%", id=723, state=TIMED_WAITING, blockCnt=0, waitCnt=23979] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6cb99fbd, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#450%DataCenter%", id=718, state=TIMED_WAITING, blockCnt=0, waitCnt=3246] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#449%DataCenter%", id=717, state=TIMED_WAITING, blockCnt=0, waitCnt=3245] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#448%DataCenter%", id=716, state=TIMED_WAITING, blockCnt=0, waitCnt=3245] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#447%DataCenter%", id=715, state=TIMED_WAITING, blockCnt=0, waitCnt=3245] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#446%DataCenter%", id=714, state=TIMED_WAITING, blockCnt=0, waitCnt=3245] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#445%DataCenter%", id=713, state=TIMED_WAITING, blockCnt=0, waitCnt=3245] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#444%DataCenter%", id=712, state=TIMED_WAITING, blockCnt=0, waitCnt=3245] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#443%DataCenter%", id=709, state=TIMED_WAITING, blockCnt=0, waitCnt=3246] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#442%DataCenter%", id=708, state=TIMED_WAITING, blockCnt=0, waitCnt=3245] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#441%DataCenter%", id=706, state=TIMED_WAITING, blockCnt=0, waitCnt=3246] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#440%DataCenter%", id=705, state=TIMED_WAITING, blockCnt=0, waitCnt=3246] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-#439%DataCenter%", id=704, state=TIMED_WAITING, blockCnt=0, waitCnt=3246] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@aa9b119, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-12", id=692, state=RUNNABLE, blockCnt=0, waitCnt=2] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@3436145b - locked java.util.Collections$UnmodifiableSet@205ed08e - locked sun.nio.ch.KQueueSelectorImpl@3ae74f at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-11", id=691, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@475cbf28 - locked java.util.Collections$UnmodifiableSet@43ccef54 - locked sun.nio.ch.KQueueSelectorImpl@15c09d03 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-10", id=690, state=RUNNABLE, blockCnt=0, waitCnt=5] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@4217106b - locked java.util.Collections$UnmodifiableSet@601042fe - locked sun.nio.ch.KQueueSelectorImpl@4482a7e7 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-9", id=689, state=RUNNABLE, blockCnt=0, waitCnt=2] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@553b12f1 - locked java.util.Collections$UnmodifiableSet@2f55c475 - locked sun.nio.ch.KQueueSelectorImpl@4d3064d3 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-8", id=686, state=RUNNABLE, blockCnt=0, waitCnt=17] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@1c5c5f0d - locked java.util.Collections$UnmodifiableSet@117d6aed - locked sun.nio.ch.KQueueSelectorImpl@100a62db at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-7", id=685, state=RUNNABLE, blockCnt=0, waitCnt=14] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@5e529af5 - locked java.util.Collections$UnmodifiableSet@20081205 - locked sun.nio.ch.KQueueSelectorImpl@5a583eeb at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-3-7", id=684, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@198793c1 - locked java.util.Collections$UnmodifiableSet@53483736 - locked sun.nio.ch.KQueueSelectorImpl@33b08f60 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-6", id=681, state=RUNNABLE, blockCnt=0, waitCnt=6] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@2884509b - locked java.util.Collections$UnmodifiableSet@d38d7d1 - locked sun.nio.ch.KQueueSelectorImpl@1f5bb0c7 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-3-6", id=680, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@28a7f7c3 - locked java.util.Collections$UnmodifiableSet@22ad7ac4 - locked sun.nio.ch.KQueueSelectorImpl@6960a71f at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-5", id=676, state=RUNNABLE, blockCnt=0, waitCnt=3] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@2b7fff7c - locked java.util.Collections$UnmodifiableSet@3c711c3f - locked sun.nio.ch.KQueueSelectorImpl@75432a2f at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-3-5", id=675, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@197733c7 - locked java.util.Collections$UnmodifiableSet@4da7bb8a - locked sun.nio.ch.KQueueSelectorImpl@7903d6e5 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-4", id=669, state=RUNNABLE, blockCnt=0, waitCnt=11] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@15fd89b8 - locked java.util.Collections$UnmodifiableSet@353469cb - locked sun.nio.ch.KQueueSelectorImpl@45bfff0c at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-3", id=668, state=RUNNABLE, blockCnt=1, waitCnt=3] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@22ecada0 - locked java.util.Collections$UnmodifiableSet@5ba5500c - locked sun.nio.ch.KQueueSelectorImpl@4d490082 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-3-4", id=667, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@7734d1b8 - locked java.util.Collections$UnmodifiableSet@30f2f353 - locked sun.nio.ch.KQueueSelectorImpl@4126e071 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-3-3", id=666, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@446ea5c7 - locked java.util.Collections$UnmodifiableSet@150615e3 - locked sun.nio.ch.KQueueSelectorImpl@32c9147b at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-2", id=664, state=RUNNABLE, blockCnt=0, waitCnt=2] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@441a4f3b - locked java.util.Collections$UnmodifiableSet@75eb9ae2 - locked sun.nio.ch.KQueueSelectorImpl@506720b1 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-3-2", id=663, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@2af79c0a - locked java.util.Collections$UnmodifiableSet@3c026df8 - locked sun.nio.ch.KQueueSelectorImpl@1597cc10 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-4-1", id=662, state=RUNNABLE, blockCnt=2, waitCnt=3] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@7c1c04bb - locked java.util.Collections$UnmodifiableSet@6a9dc0fe - locked sun.nio.ch.KQueueSelectorImpl@455f0df8 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-3-1", id=661, state=RUNNABLE, blockCnt=1, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@6e03ae0f - locked java.util.Collections$UnmodifiableSet@722646a8 - locked sun.nio.ch.KQueueSelectorImpl@7638e222 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="logback-1", id=269, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4b3501d0, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="tcp-disco-client-message-worker-[db868bb1 10.168.55.51:51438]-#12%DataCenter%", id=167, state=TIMED_WAITING, blockCnt=0, waitCnt=6057] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3835bd6f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:522) at java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:684) at o.a.i.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:8164) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at o.a.i.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:8100) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="tcp-disco-sock-reader-[db868bb1 10.168.55.51:51438 client]-#11%DataCenter%", id=166, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at sun.security.ssl.InputRecord.readFully(InputRecord.java:465) at sun.security.ssl.InputRecord.read(InputRecord.java:503) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:975) - locked java.lang.Object@7866bfc7 at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:933) at sun.security.ssl.AppInputStream.read(AppInputStream.java:105) - locked sun.security.ssl.AppInputStream@497e7b1b at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read1(BufferedInputStream.java:286) at java.io.BufferedInputStream.read(BufferedInputStream.java:345) - locked java.io.BufferedInputStream@2a26d865 at o.a.i.marshaller.jdk.JdkMarshallerInputStreamWrapper.read(JdkMarshallerInputStreamWrapper.java:52) at java.io.ObjectInputStream$PeekInputStream.read(ObjectInputStream.java:2663) at java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2679) at java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:3156) at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:862) at java.io.ObjectInputStream.(ObjectInputStream.java:358) at o.a.i.marshaller.jdk.JdkMarshallerObjectInputStream.(JdkMarshallerObjectInputStream.java:42) at o.a.i.marshaller.jdk.JdkMarshaller.unmarshal0(JdkMarshaller.java:121) at o.a.i.marshaller.AbstractNodeNameAwareMarshaller.unmarshal(AbstractNodeNameAwareMarshaller.java:92) at o.a.i.i.util.IgniteUtils.unmarshal(IgniteUtils.java:10456) at o.a.i.spi.discovery.tcp.ServerImpl$SocketReader.body(ServerImpl.java:7165) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="ttl-cleanup-worker-#94%DataCenter%", id=135, state=TIMED_WAITING, blockCnt=0, waitCnt=11781] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:7998) at o.a.i.i.processors.cache.GridCacheSharedTtlCleanupManager$CleanupWorker.body(GridCacheSharedTtlCleanupManager.java:191) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="DestroyJavaVM", id=126, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="DefaultDispatcher-worker-2", id=125, state=TIMED_WAITING, blockCnt=3, waitCnt=100] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.park(CoroutineScheduler.kt:783) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.tryPark(CoroutineScheduler.kt:728) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:711) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:665) Thread [name="DefaultDispatcher-worker-1", id=124, state=TIMED_WAITING, blockCnt=5, waitCnt=129] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.park(CoroutineScheduler.kt:783) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.tryPark(CoroutineScheduler.kt:728) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:711) at kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:665) Thread [name="srvc-deploy-#85%DataCenter%", id=123, state=WAITING, blockCnt=4, waitCnt=164] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3fedd68b, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="checkpoint-runner-#77%DataCenter%", id=114, state=WAITING, blockCnt=8, waitCnt=73530] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72b8de36, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="checkpoint-runner-#76%DataCenter%", id=113, state=WAITING, blockCnt=15, waitCnt=80471] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72b8de36, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="checkpoint-runner-#75%DataCenter%", id=112, state=WAITING, blockCnt=10, waitCnt=80576] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72b8de36, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="checkpoint-runner-#74%DataCenter%", id=111, state=WAITING, blockCnt=11, waitCnt=74284] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72b8de36, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="db-checkpoint-thread-#73%DataCenter%", id=110, state=TIMED_WAITING, blockCnt=5, waitCnt=1973] Lock [object=o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer@4824d5d9, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.waitCheckpointEvent(GridCacheDatabaseSharedManager.java:4226) at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.body(GridCacheDatabaseSharedManager.java:3711) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="dms-writer-thread-#67%DataCenter%", id=103, state=WAITING, blockCnt=0, waitCnt=21] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@545dc948, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at o.a.i.i.processors.metastorage.persistence.DmsDataWriterWorker.body(DmsDataWriterWorker.java:145) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="wal-file-compressor-%DataCenter%-2-#66%DataCenter%", id=102, state=WAITING, blockCnt=14, waitCnt=967] Lock [object=o.a.i.i.processors.cache.persistence.wal.aware.SegmentCompressStorage@5d504c6, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentCompressStorage.nextSegmentToCompressOrWait(SegmentCompressStorage.java:140) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentAware.waitNextSegmentToCompress(SegmentAware.java:110) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.tryReserveNextSegmentOrWait(FileWriteAheadLogManager.java:2180) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.body0(FileWriteAheadLogManager.java:2204) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.body(FileWriteAheadLogManager.java:2195) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="wal-file-compressor-%DataCenter%-1-#65%DataCenter%", id=101, state=WAITING, blockCnt=14, waitCnt=15] Lock [object=o.a.i.i.processors.cache.persistence.wal.aware.SegmentCompressStorage@5d504c6, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentCompressStorage.nextSegmentToCompressOrWait(SegmentCompressStorage.java:140) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentAware.waitNextSegmentToCompress(SegmentAware.java:110) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.tryReserveNextSegmentOrWait(FileWriteAheadLogManager.java:2180) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.body0(FileWriteAheadLogManager.java:2204) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.body(FileWriteAheadLogManager.java:2195) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="wal-file-decompressor%DataCenter-#64%DataCenter%", id=100, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@31282324, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:550) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileDecompressor.body(FileWriteAheadLogManager.java:2388) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="wal-file-compressor-%DataCenter%-0-#63%DataCenter%", id=99, state=WAITING, blockCnt=15, waitCnt=15] Lock [object=o.a.i.i.processors.cache.persistence.wal.aware.SegmentCompressStorage@5d504c6, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentCompressStorage.nextSegmentToCompressOrWait(SegmentCompressStorage.java:140) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentAware.waitNextSegmentToCompress(SegmentAware.java:110) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.tryReserveNextSegmentOrWait(FileWriteAheadLogManager.java:2180) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.body0(FileWriteAheadLogManager.java:2204) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressorWorker.access$3500(FileWriteAheadLogManager.java:2157) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressor.body(FileWriteAheadLogManager.java:2133) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="wal-file-archiver%DataCenter-#62%DataCenter%", id=98, state=WAITING, blockCnt=15, waitCnt=207] Lock [object=o.a.i.i.processors.cache.persistence.wal.aware.SegmentCurrentStateStorage@5fcca2c5, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentCurrentStateStorage.awaitSegment(SegmentCurrentStateStorage.java:72) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentCurrentStateStorage.waitNextSegmentForArchivation(SegmentCurrentStateStorage.java:89) at o.a.i.i.processors.cache.persistence.wal.aware.SegmentAware.waitNextSegmentForArchivation(SegmentAware.java:80) at o.a.i.i.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver.body(FileWriteAheadLogManager.java:1844) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="ignite-license-checker-DataCenter", id=96, state=TIMED_WAITING, blockCnt=2, waitCnt=109] Lock [object=java.util.TaskQueue@65e127fd, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) Thread [name="snapshot-scheduler-restats-#61%DataCenter%", id=95, state=WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@48bffcb3, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="session-timeout-worker-#48%DataCenter%", id=74, state=TIMED_WAITING, blockCnt=0, waitCnt=5918] at java.lang.Thread.sleep(Native Method) at o.a.i.i.processors.rest.GridRestProcessor$4.body(GridRestProcessor.java:516) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-3-#47%DataCenter%", id=72, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@387ec783 - locked java.util.Collections$UnmodifiableSet@21e80c3 - locked sun.nio.ch.KQueueSelectorImpl@6e2c91b2 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-2-#46%DataCenter%", id=71, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@59c1b2a0 - locked java.util.Collections$UnmodifiableSet@17cd24c4 - locked sun.nio.ch.KQueueSelectorImpl@6123a666 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-1-#45%DataCenter%", id=70, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6b081af1 - locked java.util.Collections$UnmodifiableSet@3f08f182 - locked sun.nio.ch.KQueueSelectorImpl@419d19a3 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-client-listener-0-#44%DataCenter%", id=69, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@3d719760 - locked java.util.Collections$UnmodifiableSet@1685cdad - locked sun.nio.ch.KQueueSelectorImpl@6e79ba1e at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-client-listener-#60%DataCenter%", id=94, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@4780f673 - locked java.util.Collections$UnmodifiableSet@767ba374 - locked sun.nio.ch.KQueueSelectorImpl@4e2be0b3 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3029) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2977) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="exchange-worker-#59%DataCenter%", id=93, state=TIMED_WAITING, blockCnt=2, waitCnt=1358] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6da11a22, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:522) at java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:684) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:3186) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:3111) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="disco-event-worker-#57%DataCenter%", id=91, state=WAITING, blockCnt=1, waitCnt=3168] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2dd25a8b, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:2919) at o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:2890) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="tcp-disco-ip-finder-cleaner-#8%DataCenter%", id=90, state=TIMED_WAITING, blockCnt=138, waitCnt=235] at java.lang.Thread.sleep(Native Method) at o.a.i.spi.discovery.tcp.ServerImpl$IpFinderCleaner.body(ServerImpl.java:2275) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="tcp-disco-multicast-addr-sender-#5", id=87, state=RUNNABLE, blockCnt=10, waitCnt=4] at java.net.PlainDatagramSocketImpl.receive0(Native Method) - locked java.net.PlainDatagramSocketImpl@6192c46 at java.net.AbstractPlainDatagramSocketImpl.receive(AbstractPlainDatagramSocketImpl.java:143) - locked java.net.PlainDatagramSocketImpl@6192c46 at java.net.DatagramSocket.receive(DatagramSocket.java:812) - locked java.net.DatagramPacket@17a67a4e - locked java.net.MulticastSocket@72331292 at o.a.i.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder$AddressSender.body(TcpDiscoveryMulticastIpFinder.java:889) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="tcp-disco-multicast-addr-sender-#4", id=86, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.PlainDatagramSocketImpl.receive0(Native Method) - locked java.net.PlainDatagramSocketImpl@4d0a185d at java.net.AbstractPlainDatagramSocketImpl.receive(AbstractPlainDatagramSocketImpl.java:143) - locked java.net.PlainDatagramSocketImpl@4d0a185d at java.net.DatagramSocket.receive(DatagramSocket.java:812) - locked java.net.DatagramPacket@2be76a6 - locked java.net.MulticastSocket@266e18b5 at o.a.i.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder$AddressSender.body(TcpDiscoveryMulticastIpFinder.java:889) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="tcp-disco-srvr-[:47500]-#3%DataCenter%", id=85, state=RUNNABLE, blockCnt=0, waitCnt=0] at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at sun.security.ssl.SSLServerSocketImpl.accept(SSLServerSocketImpl.java:348) at o.a.i.spi.discovery.tcp.ServerImpl$TcpServer.body(ServerImpl.java:6753) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at o.a.i.spi.discovery.tcp.ServerImpl$TcpServerThread.body(ServerImpl.java:6676) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="tcp-disco-msg-worker-[crd]-#2%DataCenter%", id=84, state=TIMED_WAITING, blockCnt=3, waitCnt=511799] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4c233968, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingDeque.pollFirst(LinkedBlockingDeque.java:522) at java.util.concurrent.LinkedBlockingDeque.poll(LinkedBlockingDeque.java:684) at o.a.i.spi.discovery.tcp.ServerImpl$MessageWorker.body(ServerImpl.java:8164) at o.a.i.spi.discovery.tcp.ServerImpl$RingMessageWorker.body(ServerImpl.java:3133) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at o.a.i.spi.discovery.tcp.ServerImpl$MessageWorkerThread.body(ServerImpl.java:8100) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="disco-notifier-worker-#56%DataCenter%", id=83, state=WAITING, blockCnt=9, waitCnt=3183] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@51f61841, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body0(GridDiscoveryManager.java:2679) at o.a.i.i.managers.discovery.GridDiscoveryManager$DiscoveryMessageNotifierWorker.body(GridDiscoveryManager.java:2724) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-data-loader-flusher-#54%DataCenter%", id=80, state=WAITING, blockCnt=0, waitCnt=1] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@208c4f9f, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.DelayQueue.take(DelayQueue.java:211) at o.a.i.i.processors.datastreamer.DataStreamProcessor$2.body(DataStreamProcessor.java:108) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-3-#52%DataCenter%", id=78, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@7671408b - locked java.util.Collections$UnmodifiableSet@234ccf54 - locked sun.nio.ch.KQueueSelectorImpl@4219ef6c at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-2-#51%DataCenter%", id=77, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2ba007 - locked java.util.Collections$UnmodifiableSet@7c22dd91 - locked sun.nio.ch.KQueueSelectorImpl@39519b2e at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-1-#50%DataCenter%", id=76, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@34be61e7 - locked java.util.Collections$UnmodifiableSet@6948d4ef - locked sun.nio.ch.KQueueSelectorImpl@36c5202 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-rest-0-#49%DataCenter%", id=75, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@36d131aa - locked java.util.Collections$UnmodifiableSet@71a7dfce - locked sun.nio.ch.KQueueSelectorImpl@5ee782b9 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-tcp-rest-#53%DataCenter%", id=79, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@3f7a3725 - locked java.util.Collections$UnmodifiableSet@990d01 - locked sun.nio.ch.KQueueSelectorImpl@4f0feb73 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3029) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2977) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="cron4j::scheduler[510b1d06306c9b2c1ab28416000001719321d6f626d77ef0]::timer[510b1d06306c9b2c1ab28416000001719321d6f77db20fe4]", id=73, state=TIMED_WAITING, blockCnt=0, waitCnt=106] at java.lang.Thread.sleep(Native Method) at it.sauronsoftware.cron4j.TimerThread.safeSleep(Unknown Source) at it.sauronsoftware.cron4j.TimerThread.run(Unknown Source) Thread [name="long-qry-#43%DataCenter%", id=68, state=TIMED_WAITING, blockCnt=0, waitCnt=5920] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils.sleep(IgniteUtils.java:7998) at o.a.i.i.processors.query.h2.LongRunningQueryManager$1.body(LongRunningQueryManager.java:92) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="mysql-cj-abandoned-connection-cleanup", id=67, state=TIMED_WAITING, blockCnt=0, waitCnt=1189] Lock [object=java.lang.ref.ReferenceQueue$Lock@2d9f5a61, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) at com.mysql.cj.jdbc.AbandonedConnectionCleanupThread.run(AbandonedConnectionCleanupThread.java:85) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.ThreadPoolExecutor$Worker@5b5b8730 Thread [name="db-snapshot-msg-executor-#42%DataCenter%", id=66, state=WAITING, blockCnt=0, waitCnt=2] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@43b283e8, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="db-snapshot-executor-stripe-1-#41%DataCenter%", id=65, state=WAITING, blockCnt=0, waitCnt=3] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="db-snapshot-executor-stripe-0-#40%DataCenter%", id=64, state=WAITING, blockCnt=0, waitCnt=3] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-2", id=62, state=TIMED_WAITING, blockCnt=0, waitCnt=100] at java.lang.Thread.sleep(Native Method) at o.a.i.i.processors.cache.persistence.diagnostic.pagelocktracker.SharedPageLockTracker$TimeOutWorker.run(SharedPageLockTracker.java:368) Thread [name="binary-metadata-writer-#39%DataCenter%", id=61, state=WAITING, blockCnt=0, waitCnt=73] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@b0b6f9a, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body0(BinaryMetadataFileStore.java:361) at o.a.i.i.processors.cache.binary.BinaryMetadataFileStore$BinaryMetadataAsyncWriter.body(BinaryMetadataFileStore.java:342) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="tcp-comm-worker-#1%DataCenter%", id=60, state=TIMED_WAITING, blockCnt=0, waitCnt=10] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3b6f59fb, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at o.a.i.spi.communication.tcp.TcpCommunicationSpi$CommunicationWorker.body(TcpCommunicationSpi.java:4554) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at o.a.i.spi.communication.tcp.TcpCommunicationSpi$4.body(TcpCommunicationSpi.java:2347) at o.a.i.spi.IgniteSpiThread.run(IgniteSpiThread.java:60) Thread [name="grid-nio-worker-tcp-comm-5-#37%DataCenter%", id=54, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@6cf450a5 - locked java.util.Collections$UnmodifiableSet@3b0e3310 - locked sun.nio.ch.KQueueSelectorImpl@6ba9fc54 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-4-#36%DataCenter%", id=53, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@2ac8c05b - locked java.util.Collections$UnmodifiableSet@6477ccdf - locked sun.nio.ch.KQueueSelectorImpl@2eabf5f2 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-3-#35%DataCenter%", id=52, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@a44f72f - locked java.util.Collections$UnmodifiableSet@66e22424 - locked sun.nio.ch.KQueueSelectorImpl@3ca90d13 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-2-#34%DataCenter%", id=51, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@4d5c6b11 - locked java.util.Collections$UnmodifiableSet@608c001a - locked sun.nio.ch.KQueueSelectorImpl@1738db2b at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-1-#33%DataCenter%", id=50, state=RUNNABLE, blockCnt=78, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@638d9d4a - locked java.util.Collections$UnmodifiableSet@4da766c9 - locked sun.nio.ch.KQueueSelectorImpl@4dab23b7 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-nio-worker-tcp-comm-0-#32%DataCenter%", id=49, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked o.a.i.i.util.nio.SelectedSelectionKeySet@1c1b48aa - locked java.util.Collections$UnmodifiableSet@13f3e1ae - locked sun.nio.ch.KQueueSelectorImpl@65944c37 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2242) at o.a.i.i.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1888) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="nio-acceptor-tcp-comm-#38%DataCenter%", id=59, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@71366da9 - locked java.util.Collections$UnmodifiableSet@37ae253d - locked sun.nio.ch.KQueueSelectorImpl@3b5c5cf7 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.accept(GridNioServer.java:3029) at o.a.i.i.util.nio.GridNioServer$GridNioAcceptWorker.body(GridNioServer.java:2977) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="grid-timeout-worker-#31%DataCenter%", id=48, state=TIMED_WAITING, blockCnt=2, waitCnt=20477] Lock [object=java.lang.Object@7c6e4a5f, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at o.a.i.i.processors.timeout.GridTimeoutProcessor$TimeoutWorker.body(GridTimeoutProcessor.java:313) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="ignite-clock", id=47, state=TIMED_WAITING, blockCnt=0, waitCnt=513137] at java.lang.Thread.sleep(Native Method) at o.a.i.i.util.IgniteUtils$25.run(IgniteUtils.java:3537) at java.lang.Thread.run(Thread.java:748) Thread [name="jvm-pause-detector-worker", id=40, state=TIMED_WAITING, blockCnt=0, waitCnt=109446] at java.lang.Thread.sleep(Native Method) at o.a.i.i.LongJVMPauseDetector$1.run(LongJVMPauseDetector.java:116) Thread [name="data-streamer-stripe-11-#24%DataCenter%", id=39, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-10-#23%DataCenter%", id=38, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-9-#22%DataCenter%", id=37, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-8-#21%DataCenter%", id=36, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-7-#20%DataCenter%", id=35, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-6-#19%DataCenter%", id=34, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-5-#18%DataCenter%", id=33, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-4-#17%DataCenter%", id=32, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-3-#16%DataCenter%", id=31, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-2-#15%DataCenter%", id=30, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-1-#14%DataCenter%", id=29, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="data-streamer-stripe-0-#13%DataCenter%", id=28, state=WAITING, blockCnt=0, waitCnt=1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-11-#12%DataCenter%", id=27, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-10-#11%DataCenter%", id=26, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-9-#10%DataCenter%", id=25, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-8-#9%DataCenter%", id=24, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-7-#8%DataCenter%", id=23, state=WAITING, blockCnt=0, waitCnt=12] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-6-#7%DataCenter%", id=22, state=WAITING, blockCnt=0, waitCnt=10] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-5-#6%DataCenter%", id=21, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-4-#5%DataCenter%", id=20, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-3-#4%DataCenter%", id=19, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-2-#3%DataCenter%", id=18, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-1-#2%DataCenter%", id=17, state=WAITING, blockCnt=0, waitCnt=8] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="sys-stripe-0-#1%DataCenter%", id=16, state=WAITING, blockCnt=0, waitCnt=7] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at o.a.i.i.util.StripedExecutor$StripeConcurrentQueue.take(StripedExecutor.java:722) at o.a.i.i.util.StripedExecutor$Stripe.body(StripedExecutor.java:545) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:119) at java.lang.Thread.run(Thread.java:748) Thread [name="nioEventLoopGroup-2-1", id=15, state=RUNNABLE, blockCnt=0, waitCnt=0] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked io.netty.channel.nio.SelectedSelectionKeySet@3ef73b5e - locked java.util.Collections$UnmodifiableSet@78b66670 - locked sun.nio.ch.KQueueSelectorImpl@21543675 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Thread [name="Thread-1", id=14, state=TIMED_WAITING, blockCnt=0, waitCnt=8517] Lock [object=java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2f66bcb2, ownerName=null, ownerId=-1] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thread [name="JVMCI-native CompilerThread2", id=10, state=RUNNABLE, blockCnt=1, waitCnt=0] Thread [name="JVMCI-native CompilerThread1", id=9, state=RUNNABLE, blockCnt=5, waitCnt=0] Thread [name="JVMCI-native CompilerThread0", id=8, state=RUNNABLE, blockCnt=4, waitCnt=0] Thread [name="JDWP Command Reader", id=7, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="JDWP Event Helper Thread", id=6, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="JDWP Transport Listener: dt_socket", id=5, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Signal Dispatcher", id=4, state=RUNNABLE, blockCnt=0, waitCnt=0] Thread [name="Finalizer", id=3, state=WAITING, blockCnt=319, waitCnt=80] Lock [object=java.lang.ref.ReferenceQueue$Lock@284c5fcb, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Thread [name="Reference Handler", id=2, state=WAITING, blockCnt=2470, waitCnt=1514] Lock [object=java.lang.ref.Reference$Lock@63023954, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) 2020-04-20 01:29:06.621 [pub-#481%DataCenter%] WARN o.a.i.i.p.c.CacheDiagnosticManager - Page locks dump: Thread=[name=DefaultDispatcher-worker-1, id=124], state=TIMED_WAITING Locked pages = [] Locked pages log: name=DefaultDispatcher-worker-1 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=checkpoint-runner-#74%DataCenter%, id=111], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#74%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=checkpoint-runner-#75%DataCenter%, id=112], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#75%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=checkpoint-runner-#76%DataCenter%, id=113], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#76%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=checkpoint-runner-#77%DataCenter%, id=114], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#77%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=db-checkpoint-thread-#73%DataCenter%, id=110], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#73%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=disco-notifier-worker-#56%DataCenter%, id=83], state=WAITING Locked pages = [] Locked pages log: name=disco-notifier-worker-#56%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=dms-writer-thread-#67%DataCenter%, id=103], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#67%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=exchange-worker-#59%DataCenter%, id=93], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#59%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=mgmt-#454%DataCenter%, id=723], state=TIMED_WAITING Locked pages = [] Locked pages log: name=mgmt-#454%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=mgmt-#456%DataCenter%, id=725], state=TIMED_WAITING Locked pages = [] Locked pages log: name=mgmt-#456%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=mgmt-#460%DataCenter%, id=729], state=TIMED_WAITING Locked pages = [] Locked pages log: name=mgmt-#460%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=mgmt-#464%DataCenter%, id=733], state=TIMED_WAITING Locked pages = [] Locked pages log: name=mgmt-#464%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-1, id=662], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-1 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-10, id=690], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-10 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-12, id=692], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-12 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-2, id=664], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-2 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-3, id=668], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-3 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-4, id=669], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-4 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-5, id=676], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-5 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-6, id=681], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-6 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-7, id=685], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-7 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-8, id=686], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-8 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=nioEventLoopGroup-4-9, id=689], state=RUNNABLE Locked pages = [] Locked pages log: name=nioEventLoopGroup-4-9 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pool-8-thread-1, id=741], state=WAITING Locked pages = [] Locked pages log: name=pool-8-thread-1 time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#459%DataCenter%, id=728], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#459%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#463%DataCenter%, id=732], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#463%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#467%DataCenter%, id=736], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#467%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#472%DataCenter%, id=744], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#472%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#475%DataCenter%, id=747], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#475%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#478%DataCenter%, id=750], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#478%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#481%DataCenter%, id=753], state=RUNNABLE Locked pages = [] Locked pages log: name=pub-#481%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#484%DataCenter%, id=756], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#484%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#487%DataCenter%, id=759], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#487%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#490%DataCenter%, id=762], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#490%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#493%DataCenter%, id=765], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#493%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=pub-#496%DataCenter%, id=768], state=TIMED_WAITING Locked pages = [] Locked pages log: name=pub-#496%DataCenter% time=(1587317346613, 2020-04-20 01:29:06.613) Thread=[name=srvc-deploy-#85%DataCenter%, id=123], state=WAITING Locked pages = [] Locked pages log: name=srvc-deploy-#85%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=sys-stripe-6-#7%DataCenter%, id=22], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-6-#7%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#457%DataCenter%, id=726], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#457%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#461%DataCenter%, id=730], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#461%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#465%DataCenter%, id=734], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#465%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#470%DataCenter%, id=742], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#470%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#473%DataCenter%, id=745], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#473%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#476%DataCenter%, id=748], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#476%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#479%DataCenter%, id=751], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#479%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#482%DataCenter%, id=754], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#482%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#485%DataCenter%, id=757], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#485%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#488%DataCenter%, id=760], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#488%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#491%DataCenter%, id=763], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#491%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614) Thread=[name=utility-#494%DataCenter%, id=766], state=TIMED_WAITING Locked pages = [] Locked pages log: name=utility-#494%DataCenter% time=(1587317346614, 2020-04-20 01:29:06.614)