From 3deab617cb24ebfd9deddd93c2e169d402d846ba Mon Sep 17 00:00:00 2001 From: heguozi Date: Wed, 17 Jun 2020 17:10:39 +0800 Subject: [PATCH] HIVE-23710: Add table meta cache limit when starting Hive server2 --- .../org/apache/hadoop/hive/conf/HiveConf.java | 4 +++- .../hive/metastore/RetryingMetaStoreClient.java | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b79515fcf0..f871853e01 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3232,7 +3232,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "1800s", new TimeValidator(TimeUnit.SECONDS), "Interval to synchronize privileges from external authorizer periodically in HS2"), - // HiveServer2 specific configs + // HiveServer2 specific configs + HIVE_SERVER2_INIT_LOAD_TABLE_LIMIT("hive.server2.init.load.table.limit", -1, + "The number of tables from a database to be inited when starting Hive server2."), HIVE_SERVER2_CLEAR_DANGLING_SCRATCH_DIR("hive.server2.clear.dangling.scratchdir", false, "Clear dangling scratch dir periodically in HS2"), HIVE_SERVER2_CLEAR_DANGLING_SCRATCH_DIR_INTERVAL("hive.server2.clear.dangling.scratchdir.interval", diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java index f97f638ba6..198c4f786d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java @@ -68,6 +68,7 @@ private final long connectionLifeTimeInMillis; private long lastConnectionTime; private boolean localMetaStore; + private final int initTableLimit; protected RetryingMetaStoreClient(Configuration conf, Class[] constructorArgTypes, @@ -89,6 +90,7 @@ protected RetryingMetaStoreClient(Configuration conf, Class[] constructorArgT this.lastConnectionTime = System.currentTimeMillis(); String msUri = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS); localMetaStore = (msUri == null) || msUri.trim().isEmpty(); + this.initTableLimit = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_INIT_LOAD_TABLE_LIMIT); reloginExpiringKeytabUser(); @@ -204,6 +206,21 @@ public Object run() throws MetaException { } } + if (initTableLimit != -1){ + if (args != null) { + int maxTableNum = 0; + for (Object o : args) { + int len = o.toString().split(",").length; + if (maxTableNum < len) { + maxTableNum = len; + } + } + if (maxTableNum > initTableLimit) { + break; + } + } + } + if (metaCallTimeMap == null) { ret = method.invoke(base, args); } else { -- 2.17.2 (Apple Git-113)