commit 1e4a016ea2efcf1c15238762b09a314bb1e99fdf Author: Nicolas Spiegelberg Date: 5 weeks ago Improve recovery time of the HBase client when a region server dies. Summary: When a region server dies, the HBase client waits until the RPC timesout before learning that it needs to check META to find the new location of the region. And it incurs this *timeout* cost for every region being served by the dead region server. This diff fixes this by clearing the entries in cache that have the dead region server as their values. Test Plan: I wrote a interative client program that inserts to two rows in two different regions served by the same region server. While this client is waiting for user input after the insertions, I power off this region server from radium. After the regions originally served by the dead server are re-distributed to other region servers, I tell the client program to perform gets on the two rows. I saw only one RPC timeout rather than two. diff --git src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java index cba7bd1..17bb49a 100644 --- src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -23,7 +23,9 @@ import java.io.Closeable; import java.io.IOException; import java.lang.reflect.Proxy; import java.lang.reflect.UndeclaredThrowableException; +import java.net.ConnectException; import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -31,8 +33,8 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Map.Entry; +import java.util.NoSuchElementException; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Callable; @@ -475,6 +477,13 @@ public class HConnectionManager { cachedRegionLocations = new HashMap>(); + // The presence of a server in the map implies it's likely that there is an + // entry in cachedRegionLocations that map to this server; but the absence + // of a server in this map guarentees that there is no entry in cache that + // maps to the absent server. + private final Set cachedServers = + new HashSet(); + // region cache prefetch is enabled by default. this set contains all // tables whose region cache prefetch are disabled. private final Set regionCachePrefetchDisabledTables = @@ -1078,6 +1087,35 @@ public class HConnectionManager { } /* + * Delete all cached entries of a table that maps to a specific location. + * + * @param tablename + * @param server + */ + private void clearCachedLocationForServer( + final String server) { + boolean deletedSomething = false; + synchronized (this.cachedRegionLocations) { + if (!cachedServers.contains(server)) { + return; + } + for (SoftValueSortedMap tableLocations : + cachedRegionLocations.values()) { + for (Entry e : tableLocations.entrySet()) { + if (e.getValue().getServerAddress().toString().equals(server)) { + tableLocations.remove(e.getKey()); + deletedSomething = true; + } + } + } + cachedServers.remove(server); + } + if (deletedSomething && LOG.isDebugEnabled()) { + LOG.debug("Removed all cached region locations that map to " + server); + } + } + + /* * @param tableName * @return Map of cached locations for passed tableName */ @@ -1102,6 +1140,7 @@ public class HConnectionManager { public void clearRegionCache() { synchronized(this.cachedRegionLocations) { this.cachedRegionLocations.clear(); + this.cachedServers.clear(); } } @@ -1120,7 +1159,12 @@ public class HConnectionManager { byte [] startKey = location.getRegionInfo().getStartKey(); SoftValueSortedMap tableLocations = getTableLocations(tableName); - if (tableLocations.put(startKey, location) == null) { + boolean hasNewCache = false; + synchronized (this.cachedRegionLocations) { + cachedServers.add(location.getServerAddress().toString()); + hasNewCache = (tableLocations.put(startKey, location) == null); + } + if (hasNewCache) { LOG.debug("Cached location for " + location.getRegionInfo().getRegionNameAsString() + " is " + location.getHostnamePort()); @@ -1243,6 +1287,17 @@ public class HConnectionManager { } catch (Throwable t) { callable.shouldRetry(t); t = translateException(t); + if (t instanceof SocketTimeoutException || + t instanceof ConnectException || + t instanceof RetriesExhaustedException) { + // if thrown these exceptions, we clear all the cache entries that + // map to that slow/dead server; otherwise, let cache miss and ask + // .META. again to find the new location + HRegionLocation hrl = callable.location; + if (hrl != null) { + clearCachedLocationForServer(hrl.getServerAddress().toString()); + } + } RetriesExhaustedException.ThrowableWithExtraContext qt = new RetriesExhaustedException.ThrowableWithExtraContext(t, System.currentTimeMillis(), callable.toString());