[2020-06-04 11:04:52,769][ERROR][sys-stripe-6-#7][root] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureHandler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.StorageException: Failed to initialize partition file: /home/ranger/EIIP/tools/work/db/ServerNode/cache-TOFTableCache/part-942.bin]] class org.apache.ignite.internal.processors.cache.persistence.StorageException: Failed to initialize partition file: /home/ranger/EIIP/tools/work/db/ServerNode/cache-TOFTableCache/part-942.bin at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore.init(FilePageStore.java:553) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore.read(FilePageStore.java:437) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:521) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.read(FilePageStoreManager.java:505) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:880) at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:707) at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.getOrAllocatePartitionMetas(GridCacheOffheapManager.java:1909) at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.init0(GridCacheOffheapManager.java:1693) at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.invoke(GridCacheOffheapManager.java:2442) at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.invoke(IgniteCacheOffheapManagerImpl.java:445) at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerUpdate(GridCacheMapEntry.java:2314) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateSingle(GridDhtAtomicCache.java:2641) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.update(GridDhtAtomicCache.java:2102) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1919) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1719) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateRequest(GridDhtAtomicCache.java:3306) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$400(GridDhtAtomicCache.java:141) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:273) at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:268) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1847) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1472) at org.apache.ignite.internal.managers.communication.GridIoManager.access$5200(GridIoManager.java:229) at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1367) at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:565) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) Caused by: java.nio.file.FileSystemException: /home/ranger/EIIP/tools/work/db/ServerNode/cache-TOFTableCache/part-942.bin: Too many open files at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.newAsynchronousFileChannel(UnixFileSystemProvider.java:196) at java.nio.channels.AsynchronousFileChannel.open(AsynchronousFileChannel.java:248) at java.nio.channels.AsynchronousFileChannel.open(AsynchronousFileChannel.java:301) at org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIO.(AsyncFileIO.java:66) at org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory.create(AsyncFileIOFactory.java:44) at org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore.init(FilePageStore.java:523) ... 31 more [2020-06-04 11:04:52,867][WARN ][sys-stripe-6-#7][CacheDiagnosticManager] Page locks dump: Thread=[name=checkpoint-runner-#53, id=98], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#53 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=checkpoint-runner-#54, id=99], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#54 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=checkpoint-runner-#55, id=100], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#55 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=checkpoint-runner-#56, id=101], state=WAITING Locked pages = [] Locked pages log: name=checkpoint-runner-#56 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=db-checkpoint-thread-#52, id=97], state=TIMED_WAITING Locked pages = [] Locked pages log: name=db-checkpoint-thread-#52 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=dms-writer-thread-#46, id=91], state=WAITING Locked pages = [] Locked pages log: name=dms-writer-thread-#46 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=exchange-worker-#41, id=85], state=TIMED_WAITING Locked pages = [] Locked pages log: name=exchange-worker-#41 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=query-#163, id=216], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#163 time=(1591239892797, 2020-06-04 11:04:52.797) Thread=[name=query-#164, id=217], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#164 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=query-#165, id=218], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#165 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=query-#166, id=219], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#166 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=query-#167, id=220], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#167 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=query-#168, id=221], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#168 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=query-#169, id=222], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#169 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=query-#170, id=223], state=TIMED_WAITING Locked pages = [] Locked pages log: name=query-#170 time=(1591239892801, 2020-06-04 11:04:52.801) Thread=[name=sys-stripe-0-#1, id=9], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-0-#1 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=sys-stripe-1-#2, id=10], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-1-#2 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=sys-stripe-2-#3, id=11], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-2-#3 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=sys-stripe-3-#4, id=12], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-3-#4 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=sys-stripe-4-#5, id=13], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-4-#5 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=sys-stripe-5-#6, id=14], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-5-#6 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=sys-stripe-6-#7, id=15], state=RUNNABLE Locked pages = [] Locked pages log: name=sys-stripe-6-#7 time=(1591239892796, 2020-06-04 11:04:52.796) Thread=[name=sys-stripe-7-#8, id=16], state=WAITING Locked pages = [] Locked pages log: name=sys-stripe-7-#8 time=(1591239892796, 2020-06-04 11:04:52.796) [2020-06-04 11:04:52,888][ERROR][sys-stripe-6-#7][root] JVM will be halted immediately due to the failure: [failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.StorageException: Failed to initialize partition file: /home/ranger/EIIP/tools/work/db/ServerNode/cache-TOFTableCache/part-942.bin]]