diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 72e4ff2..6c53888 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -119,6 +119,7 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, + HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, HiveConf.ConfVars.METASTOREPWD, HiveConf.ConfVars.METASTORECONNECTURLHOOK, HiveConf.ConfVars.METASTORECONNECTURLKEY, @@ -398,6 +399,11 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", "600s", new TimeValidator(TimeUnit.SECONDS), "MetaStore Client socket timeout in seconds"), + METASTORE_CLIENT_SOCKET_LIFETIME("hive.metastore.client.socket.lifetime", "0s", + new TimeValidator(TimeUnit.SECONDS), + "MetaStore Client socket lifetime in seconds. After this time is exceeded, client\n" + + "reconnects on the next MetaStore operation. A value of 0s means the connection\n" + + "has an infinite lifetime."), METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine", "password to use against metastore database"), METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", "", diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 130fd67..dffeb34 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -2867,4 +2867,34 @@ private void createFunction(String dbName, String funcName, String className, ownerName, ownerType, createTime, functionType, resources); client.createFunction(func); } + + public void testRetriableClientWithConnLifetime() throws Exception { + + HiveConf conf = new HiveConf(hiveConf); + conf.setLong(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME.name(), 60); + long timeout = 65 * 1000; // Lets use a timeout more than the socket lifetime to simulate a reconnect + + // Test a normal retriable client + IMetaStoreClient client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName()); + client.getAllDatabases(); + client.close(); + + // Connect after the lifetime, there should not be any failures + client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName()); + Thread.sleep(timeout); + client.getAllDatabases(); + client.close(); + } + + private HiveMetaHookLoader getHookLoader() { + HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { + @Override + public HiveMetaHook getHook( + org.apache.hadoop.hive.metastore.api.Table tbl) + throws MetaException { + return null; + } + }; + return hookLoader; + } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java index 5ce58ee..40f3257 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java @@ -52,9 +52,9 @@ private final int retryLimit; private final long retryDelaySeconds; private final Map metaCallTimeMap; - - - + private final long connectionLifeTimeInMillis; + private long lastConnectionTime; + private boolean localMetaStore; protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader, Map metaCallTimeMap, Class msClientClass) throws MetaException { @@ -62,6 +62,11 @@ protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoad this.retryDelaySeconds = hiveConf.getTimeVar( HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); this.metaCallTimeMap = metaCallTimeMap; + this.connectionLifeTimeInMillis = + hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS) * 1000; + this.lastConnectionTime = System.currentTimeMillis(); + String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS); + localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] { @@ -94,8 +99,9 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl while (true) { try { reloginExpiringKeytabUser(); - if(retriesMade > 0){ + if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) { base.reconnect(); + lastConnectionTime = System.currentTimeMillis(); } if (metaCallTimeMap == null) { ret = method.invoke(base, args); @@ -157,6 +163,19 @@ private String getMethodString(Method method) { return methodSb.toString(); } + private boolean hasConnectionLifeTimeReached(Method method) { + if (connectionLifeTimeInMillis <= 0 || localMetaStore || + method.getName().equalsIgnoreCase("close")) { + return false; + } + boolean shouldReconnect = + (System.currentTimeMillis() - lastConnectionTime) >= connectionLifeTimeInMillis; + if (LOG.isDebugEnabled()) { + LOG.debug("Reconnection status for Method: " + method.getName() + " is " + shouldReconnect); + } + return shouldReconnect; + } + /** * Relogin if login user is logged in using keytab * Relogin is actually done by ugi code only if sufficient time has passed