[IOTDB-2413] iotdb：The file handle is not released

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 0.13.0-SNAPSHOT
• Fix Version/s: 0.13.0
• Component/s: Core/Server
• Labels: 0.13.0 pull-request-available
• Sprint: 2022-1-Tian
• Assignee: Yuan Tian
• Reporter: 刘珍
• Created: 17/Jan/22 02:46
• Updated: 28/Jan/22 01:54
• Resolved: 20/Jan/22 11:38

Description

master 0114 302e9b9153ec1d2c39187151a425ffac9275f6b6
问题现象：
长测运行（读写混合，配置文件见附件） 45小时
iotdb Too many open files
lsof -p iotdb_pid大量tsfile文件句柄没释放。

cat 1.out |grep "0.tsfile"|wc -l
65213

2022-01-16 15:34:56,838 [pool-170-IoTDB-Query-1] ERROR o.a.i.d.q.d.RawQueryDataSetWithoutValueFilter:571 - exception happened in producer thread
java.lang.NullPointerException: null
at org.apache.iotdb.tsfile.read.TsFileSequenceReader.<init>(TsFileSequenceReader.java:137)
at org.apache.iotdb.tsfile.read.TsFileSequenceReader.<init>(TsFileSequenceReader.java:117)
at org.apache.iotdb.db.query.control.FileReaderManager.get(FileReaderManager.java:121)
at org.apache.iotdb.db.engine.cache.BloomFilterCache.lambda$new$1(BloomFilterCache.java:73)
at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:141)
at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139)
at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2405)
at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853)
at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2403)
at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2386)
at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108)
at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:54)
at org.apache.iotdb.db.engine.cache.BloomFilterCache.get(BloomFilterCache.java:99)
at org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache.get(TimeSeriesMetadataCache.java:177)
at org.apache.iotdb.db.utils.FileLoaderUtils.loadTimeSeriesMetadata(FileLoaderUtils.java:107)
at org.apache.iotdb.db.query.reader.series.SeriesReader.loadTimeSeriesMetadata(SeriesReader.java:1103)
at org.apache.iotdb.db.query.reader.series.SeriesReader.unpackSeqTsFileResource(SeriesReader.java:1069)
at org.apache.iotdb.db.query.reader.series.SeriesReader.tryToUnpackAllOverlappedFilesToTimeSeriesMetadata(SeriesReader.java:993)
at org.apache.iotdb.db.query.reader.series.SeriesReader.hasNextFile(SeriesReader.java:263)
at org.apache.iotdb.db.query.reader.series.SeriesReader.hasNextChunk(SeriesReader.java:336)
at org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader.readChunkData(SeriesRawDataBatchReader.java:170)
at org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader.hasNextBatch(SeriesRawDataBatchReader.java:118)
at org.apache.iotdb.db.query.dataset.RawQueryDataSetWithoutValueFilter$ReadTask.runMayThrow(RawQueryDataSetWithoutValueFilter.java:96)
at org.apache.iotdb.db.concurrent.WrappedRunnable.run(WrappedRunnable.java:32)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2022-01-16 15:34:56,841 [pool-170-IoTDB-Query-8] ERROR o.a.i.t.f.f.LocalFSInputFactory:40 - Failed to get TsFile input of file: /data/iotdb_data/data/sequence/root.test.g_41/0/0/1642317771628-4999-0-0.tsfile,
java.nio.file.FileSystemException: /data/iotdb_data/data/sequence/root.test.g_41/0/0/1642317771628-4999-0-0.tsfile: Too many open files
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177)
at java.nio.channels.FileChannel.open(FileChannel.java:287)
at java.nio.channels.FileChannel.open(FileChannel.java:335)
at org.apache.iotdb.tsfile.read.reader.LocalTsFileInput.<init>(LocalTsFileInput.java:42)
at org.apache.iotdb.tsfile.fileSystem.fileInputFactory.LocalFSInputFactory.getTsFileInput(LocalFSInputFactory.java:38)
at org.apache.iotdb.tsfile.read.TsFileSequenceReader.<init>(TsFileSequenceReader.java:131)
at org.apache.iotdb.tsfile.read.TsFileSequenceReader.<init>(TsFileSequenceReader.java:117)
at org.apache.iotdb.db.query.control.FileReaderManager.get(FileReaderManager.java:121)
at org.apache.iotdb.db.engine.cache.BloomFilterCache.lambda$new$1(BloomFilterCache.java:73)
at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:141)
at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139)
at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2405)
at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1892)
at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2403)
at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2386)
at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108)
at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:54)
at org.apache.iotdb.db.engine.cache.BloomFilterCache.get(BloomFilterCache.java:99)
at org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache.get(TimeSeriesMetadataCache.java:177)
at org.apache.iotdb.db.utils.FileLoaderUtils.loadTimeSeriesMetadata(FileLoaderUtils.java:107)
at org.apache.iotdb.db.query.reader.series.SeriesReader.loadTimeSeriesMetadata(SeriesReader.java:1103)
at org.apache.iotdb.db.query.reader.series.SeriesReader.unpackSeqTsFileResource(SeriesReader.java:1069)
at org.apache.iotdb.db.query.reader.series.SeriesReader.tryToUnpackAllOverlappedFilesToTimeSeriesMetadata(SeriesReader.java:993)
at org.apache.iotdb.db.query.reader.series.SeriesReader.hasNextFile(SeriesReader.java:263)
at org.apache.iotdb.db.query.reader.series.SeriesReader.hasNextChunk(SeriesReader.java:336)
at org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader.readChunkData(SeriesRawDataBatchReader.java:170)
at org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader.hasNextBatch(SeriesRawDataBatchReader.java:118)
at org.apache.iotdb.tsfile.read.query.timegenerator.node.LeafNode.hasNext(LeafNode.java:51)
at org.apache.iotdb.tsfile.read.query.timegenerator.TimeGenerator.hasNext(TimeGenerator.java:53)
at org.apache.iotdb.db.query.executor.AggregationExecutor.aggregateWithValueFilter(AggregationExecutor.java:713)
at org.apache.iotdb.db.query.executor.AggregationExecutor.executeWithValueFilter(AggregationExecutor.java:661)
at org.apache.iotdb.db.query.executor.QueryRouter.aggregate(QueryRouter.java:152)
at org.apache.iotdb.db.qp.executor.PlanExecutor.processDataQuery(PlanExecutor.java:617)
at org.apache.iotdb.db.qp.executor.PlanExecutor.processQuery(PlanExecutor.java:254)
at org.apache.iotdb.db.service.basic.ServiceProvider.createQueryDataSet(ServiceProvider.java:252)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl.executeQueryPlan(TSServiceImpl.java:751)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl.access$000(TSServiceImpl.java:160)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl$QueryTask.call(TSServiceImpl.java:218)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl$QueryTask.call(TSServiceImpl.java:162)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

Issue Links

• fixes: IOTDB-2426 WAL deadlock caused by too many open files
• links to: GitHub Pull Request #4889