diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6d2748e..bfe9be3 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -385,6 +385,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", "600s", new TimeValidator(TimeUnit.SECONDS), "MetaStore Client socket timeout in seconds"), + METASTORE_CLIENT_SOCKET_LIFETIME("hive.metastore.client.socket.lifetime", "10s", + new TimeValidator(TimeUnit.SECONDS), + "MetaStore Client socket lifetime in seconds"), METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine", "password to use against metastore database"), METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", "", diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 130fd67..dffeb34 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -2867,4 +2867,34 @@ private void createFunction(String dbName, String funcName, String className, ownerName, ownerType, createTime, functionType, resources); client.createFunction(func); } + + public void testRetriableClientWithConnLifetime() throws Exception { + + HiveConf conf = new HiveConf(hiveConf); + conf.setLong(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME.name(), 60); + long timeout = 65 * 1000; // Lets use a timeout more than the socket lifetime to simulate a reconnect + + // Test a normal retriable client + IMetaStoreClient client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName()); + client.getAllDatabases(); + client.close(); + + // Connect after the lifetime, there should not be any failures + client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName()); + Thread.sleep(timeout); + client.getAllDatabases(); + client.close(); + } + + private HiveMetaHookLoader getHookLoader() { + HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { + @Override + public HiveMetaHook getHook( + org.apache.hadoop.hive.metastore.api.Table tbl) + throws MetaException { + return null; + } + }; + return hookLoader; + } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java index b4f02fc..3010073 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java @@ -51,6 +51,9 @@ private final IMetaStoreClient base; private final int retryLimit; private final long retryDelaySeconds; + private final long connectionLifeTimeInMillis; + private long lastConnectionTime; + private boolean localMetaStore; @@ -59,6 +62,11 @@ protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoad this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES); this.retryDelaySeconds = hiveConf.getTimeVar( HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); + this.connectionLifeTimeInMillis = + hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS) * 1000; + this.lastConnectionTime = System.currentTimeMillis(); + String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS); + localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] { @@ -85,8 +93,9 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl while (true) { try { reloginExpiringKeytabUser(); - if(retriesMade > 0){ + if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) { base.reconnect(); + lastConnectionTime = System.currentTimeMillis(); } ret = method.invoke(base, args); break; @@ -116,6 +125,17 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl return ret; } + private boolean hasConnectionLifeTimeReached(Method method) { + if (connectionLifeTimeInMillis <= 0 || localMetaStore || + method.getName().equalsIgnoreCase("close")) { + return false; + } + boolean shouldReconnect = + (System.currentTimeMillis() - lastConnectionTime) >= connectionLifeTimeInMillis; + LOG.info("Reconnection status for Method: " + method.getName() + " is " + shouldReconnect); + return shouldReconnect; + } + /** * Relogin if login user is logged in using keytab * Relogin is actually done by ugi code only if sufficient time has passed