Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision 101224) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision 101225) @@ -26,20 +26,13 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; -import java.net.InetAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.UnknownHostException; +import java.net.*; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import javax.security.auth.login.LoginException; @@ -156,6 +149,9 @@ private static final long THIRTY_SECONDS = 30 * 1000; private static final int RETRY_LIMIT = 2; + private static final Map blackListMetaStore = new ConcurrentHashMap(); + private static final long MAX_KEEP_TIMES = 10000l; + public HiveMetaStoreClient(HiveConf conf) throws MetaException { this(conf, null); @@ -367,6 +363,18 @@ for (int attempt = 0; !isConnected && attempt < retries; ++attempt) { for (URI store : metastoreUris) { + //filter out blacklist URIs + if (blackListMetaStore.containsKey(store)) { + AtomicLong curCount = blackListMetaStore.get(store); + if (curCount.incrementAndGet() >= MAX_KEEP_TIMES) { + LOG.warn("removing blacklisted metastore for retry : " + store); + blackListMetaStore.remove(store); + } + if (blackListMetaStore.containsKey(store)) { + LOG.warn("Blacklisted metastore with URI " + store + " ignoring..."); + continue; + } + } LOG.info("Trying to connect to metastore with URI " + store); try { transport = new TSocket(store.getHost(), store.getPort(), clientSocketTimeout); @@ -409,6 +417,15 @@ transport.open(); isConnected = true; } catch (TTransportException e) { + + //add to blacklist + if (e.getCause() != null && e.getCause() instanceof ConnectException + && e.getMessage().contains("timed out")) { + if (!blackListMetaStore.containsKey(store)) { + LOG.warn("SocketTimeout when connect to " + store + ".Adding it to blacklist."); + blackListMetaStore.put(store, new AtomicLong(0)); + } + } tte = e; if (LOG.isDebugEnabled()) { LOG.warn("Failed to connect to the MetaStore Server...", e);