diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index c19d29f..74ac59c 100644
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -54,7 +54,7 @@ public class HiveConf extends Configuration {
HiveConf.ConfVars.METASTOREDIRECTORY,
HiveConf.ConfVars.METASTOREWAREHOUSE,
HiveConf.ConfVars.METASTOREURIS,
- HiveConf.ConfVars.METATORETHRIFTRETRIES,
+ HiveConf.ConfVars.METASTORETHRIFTRETRIES,
HiveConf.ConfVars.METASTOREPWD,
HiveConf.ConfVars.METASTORECONNECTURLHOOK,
HiveConf.ConfVars.METASTORECONNECTURLKEY,
@@ -128,7 +128,11 @@ public class HiveConf extends Configuration {
METASTOREWAREHOUSE("hive.metastore.warehouse.dir", ""),
METASTOREURIS("hive.metastore.uris", ""),
// Number of times to retry a connection to a Thrift metastore server
- METATORETHRIFTRETRIES("hive.metastore.connect.retries", 3),
+ METASTORETHRIFTRETRIES("hive.metastore.connect.retries", 5),
+ // Number of seconds the client should wait between connection attempts
+ METASTORE_CLIENT_CONNECT_RETRY_DELAY("hive.metastore.client.connect.retry.delay", 1),
+ // Socket timeout for the client connection (in seconds)
+ METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", 20),
METASTOREPWD("javax.jdo.option.ConnectionPassword", ""),
// Class name of JDO connection url hook
METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", ""),
@@ -334,7 +338,7 @@ public class HiveConf extends Configuration {
HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false),
HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false),
-
+
HIVE_INDEX_IGNORE_HDFS_LOC("hive.index.compact.file.ignore.hdfs", false),
;
diff --git conf/hive-default.xml conf/hive-default.xml
index 7662f11..226719a 100644
--- conf/hive-default.xml
+++ conf/hive-default.xml
@@ -211,6 +211,18 @@
+ hive.metastore.client.connect.retry.delay
+ 1
+ Number of seconds for the client to wait between consecutive connection attempts
+
+
+
+ hive.metastore.client.socket.timeout
+ 20
+ MetaStore Client socket timeout in seconds
+
+
+
hive.metastore.rawstore.impl
org.apache.hadoop.hive.metastore.ObjectStore
Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database
diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 720c1d0..d099d95 100644
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -35,6 +35,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -65,7 +66,7 @@ import org.apache.thrift.transport.TTransportException;
public class HiveMetaStoreClient implements IMetaStoreClient {
ThriftHiveMetastore.Iface client = null;
private TTransport transport = null;
- private boolean open = false;
+ private boolean isConnected = false;
private URI metastoreUris[];
private final boolean standAloneClient = false;
private final HiveMetaHookLoader hookLoader;
@@ -73,6 +74,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
// for thrift connects
private int retries = 5;
+ private int retryDelaySeconds = 0;
static final private Log LOG = LogFactory.getLog("hive.metastore");
@@ -95,12 +97,13 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
// instantiate the metastore server handler directly instead of connecting
// through the network
client = new HiveMetaStore.HMSHandler("hive client", conf);
- open = true;
+ isConnected = true;
return;
}
// get the number retries
- retries = HiveConf.getIntVar(conf, HiveConf.ConfVars.METATORETHRIFTRETRIES);
+ retries = HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORETHRIFTRETRIES);
+ retryDelaySeconds = conf.getIntVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY);
// user wants file store based configuration
if (conf.getVar(HiveConf.ConfVars.METASTOREURIS) != null) {
@@ -161,13 +164,13 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
openStore(store);
} catch (MetaException e) {
LOG.warn(e.getStackTrace());
- LOG.warn("Unable to connect metastore with URI " + store);
+ LOG.warn("Unable to connect to metastore with URI " + store);
}
- if (open) {
+ if (isConnected) {
break;
}
}
- if (!open) {
+ if (!isConnected) {
throw new MetaException(
"Could not connect to meta store using any of the URIs provided");
}
@@ -175,11 +178,19 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
}
private void openStore(URI store) throws MetaException {
+ isConnected = false;
+
+ for (int attempt = 0; !isConnected && attempt < retries; ++attempt) {
+ if (attempt > 0 && retryDelaySeconds > 0) {
+ try {
+ LOG.info("Waiting " + retryDelaySeconds + " seconds before next connection attempt.");
+ Thread.sleep(retryDelaySeconds * 1000);
+ } catch (InterruptedException ignore) {
+ }
+ }
- for (int i = 0; i < retries && !open; ++i) {
- open = false;
transport = new TSocket(store.getHost(), store.getPort());
- ((TSocket) transport).setTimeout(20000);
+ ((TSocket)transport).setTimeout(1000 * conf.getIntVar(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT));
// Wrap thrift connection with SASL if enabled.
boolean useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL);
@@ -189,7 +200,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
ShimLoader.getHadoopThriftAuthBridge().createClient();
String principalConfig = conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL);
transport = authBridge.createClientTransport(
- principalConfig, store.getHost(), "KERBEROS",transport);
+ principalConfig, store.getHost(), "KERBEROS", transport);
} catch (IOException ioe) {
LOG.error("Couldn't create client transport", ioe);
throw new MetaException(ioe.toString());
@@ -201,28 +212,23 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
try {
transport.open();
- open = true;
+ isConnected = true;
} catch (TTransportException e) {
if (LOG.isDebugEnabled()) {
- LOG.warn("failed to connect to MetaStore, re-trying...", e);
+ LOG.warn("Failed to connect to the MetaStore Server...", e);
} else {
- // Don't print full exception trace if DEBUG is not on.
- LOG.warn("failed to connect to MetaStore, re-trying...");
- }
-
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ignore) {
+ // Don't print full exception trace if DEBUG is not on.
+ LOG.warn("Failed to connect to the MetaStore Server...");
}
}
}
- if (!open) {
- throw new MetaException("could not connect to meta store");
+ if (!isConnected) {
+ throw new MetaException("Could not connect to the MetaStore server!");
}
}
public void close() {
- open = false;
+ isConnected = false;
if ((transport != null) && transport.isOpen()) {
transport.close();
}
@@ -962,5 +968,5 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
}
}
}
-
+
}
diff --git metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
index 57648b6..60a8e8a 100644
--- metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
+++ metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
@@ -19,20 +19,23 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
private static final String METASTORE_PORT = "29083";
- private static boolean isServerRunning = false;
+ private static boolean isServerStarted = false;
+
+ public TestRemoteHiveMetaStore() {
+ super();
+ isThriftClient = true;
+ }
private static class RunMS implements Runnable {
@Override
public void run() {
- System.out.println("Running metastore!");
- String [] args = new String [1];
- args[0] = METASTORE_PORT;
- HiveMetaStore.main(args);
+ HiveMetaStore.main(new String[] { METASTORE_PORT });
}
}
@@ -40,11 +43,16 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
@Override
protected void setUp() throws Exception {
super.setUp();
- if(isServerRunning) {
+
+ if (isServerStarted) {
+ assertNotNull("Unable to connect to the MetaStore server", client);
return;
}
+
+ System.out.println("Starting MetaStore Server on port " + METASTORE_PORT);
Thread t = new Thread(new RunMS());
t.start();
+ isServerStarted = true;
// Wait a little bit for the metastore to start. Should probably have
// a better way of detecting if the metastore has started?
@@ -53,13 +61,10 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
// hive.metastore.local should be defined in HiveConf
hiveConf.set("hive.metastore.local", "false");
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + METASTORE_PORT);
- hiveConf.setIntVar(HiveConf.ConfVars.METATORETHRIFTRETRIES, 3);
+ hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3);
+ hiveConf.setIntVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, 60);
client = new HiveMetaStoreClient(hiveConf);
- isThriftClient = true;
-
- // Now you have the client - run necessary tests.
- isServerRunning = true;
}
}