[IOTDB-3014] [Rocksdb_based] OOM

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 0.14.0-SNAPSHOT
• Component/s: Core/Schema Manager
• Labels: pull-request-available
• Assignee: Jianyun Cheng
• Reporter: 刘珍
• Created: 26/Apr/22 09:33
• Updated: 30/May/22 02:42
• Resolved: 26/Apr/22 14:52

Description

master_0426_c22c140

8C32G ,堆内/堆外默认，schema_engine_mode=Rocksdb_based
DEVICE_NUMBER=50
SENSOR_NUMBER=200000
GROUP_NUMBER=10

benchmark执行写入顺序数据（配置文件见附件）

iotdb报错（日志见附件）：
2022-04-26 15:44:48,684 [pool-13-IoTDB-RPC-Client-10] WARN o.a.i.d.u.ErrorHandlingUtils:61 - Status code: INTERNAL_SERVER_ERROR(500), operation: executeNonQueryPlan failed
java.lang.UnsupportedOperationException: null
at org.apache.iotdb.db.metadata.schemaregion.rocksdb.mnode.RMeasurementMNode.getLastCacheContainer(RMeasurementMNode.java:145)
at org.apache.iotdb.db.metadata.lastCache.LastCacheManager.updateLastCache(LastCacheManager.java:88)
at org.apache.iotdb.db.metadata.LocalSchemaProcessor.updateLastCache(LocalSchemaProcessor.java:1163)
at org.apache.iotdb.db.engine.storagegroup.DataRegion.tryToUpdateBatchInsertLastCache(DataRegion.java:1231)
at org.apache.iotdb.db.engine.storagegroup.DataRegion.insertTablet(DataRegion.java:974)
at org.apache.iotdb.db.engine.StorageEngine.insertTablet(StorageEngine.java:663)
at org.apache.iotdb.db.qp.executor.PlanExecutor.insertTablet(PlanExecutor.java:1905)
at org.apache.iotdb.db.qp.executor.PlanExecutor.processNonQuery(PlanExecutor.java:309)
at org.apache.iotdb.db.service.basic.StandaloneServiceProvider.executeNonQuery(StandaloneServiceProvider.java:53)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl.executeNonQueryPlan(TSServiceImpl.java:2066)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl.insertTablet(TSServiceImpl.java:1576)
at org.apache.iotdb.service.rpc.thrift.TSIService$Processor$insertTablet.getResult(TSIService.java:3328)
at org.apache.iotdb.service.rpc.thrift.TSIService$Processor$insertTablet.getResult(TSIService.java:3308)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:38)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:248)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

2022-04-26 15:46:10,208 [pool-13-IoTDB-RPC-Client-7] ERROR o.a.i.c.c.IoTDBDefaultThreadExceptionHandler:31 - Exception in thread pool-13-IoTDB-RPC-Client-7-36
java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.rocksdb.RocksDB.keyMayExistFoundValue(Native Method)
at org.rocksdb.RocksDB.keyMayExist(RocksDB.java:2800)
at org.rocksdb.RocksDB.keyMayExist(RocksDB.java:2663)
at org.rocksdb.RocksDB.keyMayExist(RocksDB.java:2601)
at org.rocksdb.RocksDB.keyMayExist(RocksDB.java:2570)
at org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaReadWriteHandler.keyExist(RSchemaReadWriteHandler.java:299)
at org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaReadWriteHandler.keyExistByType(RSchemaReadWriteHandler.java:254)
at org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaRegion.getMeasurementMNode(RSchemaRegion.java:1229)
at org.apache.iotdb.db.metadata.schemaregion.rocksdb.RSchemaRegion.getSeriesSchemasAndReadLockDevice(RSchemaRegion.java:1742)
at org.apache.iotdb.db.metadata.LocalSchemaProcessor.getSeriesSchemasAndReadLockDevice(LocalSchemaProcessor.java:1259)
at org.apache.iotdb.db.engine.StorageEngine.getSeriesSchemas(StorageEngine.java:1060)
at org.apache.iotdb.db.engine.StorageEngine.insertTablet(StorageEngine.java:662)
at org.apache.iotdb.db.qp.executor.PlanExecutor.insertTablet(PlanExecutor.java:1905)
at org.apache.iotdb.db.qp.executor.PlanExecutor.processNonQuery(PlanExecutor.java:309)
at org.apache.iotdb.db.service.basic.StandaloneServiceProvider.executeNonQuery(StandaloneServiceProvider.java:53)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl.executeNonQueryPlan(TSServiceImpl.java:2066)
at org.apache.iotdb.db.service.thrift.impl.TSServiceImpl.insertTablet(TSServiceImpl.java:1576)
at org.apache.iotdb.service.rpc.thrift.TSIService$Processor$insertTablet.getResult(TSIService.java:3328)
at org.apache.iotdb.service.rpc.thrift.TSIService$Processor$insertTablet.getResult(TSIService.java:3308)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:38)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:248)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

Issue Links

• links to: GitHub Pull Request #5677