commit 101425b63391ccca38b550e7f8710706e9e95261 Author: nspiegelberg Date: 30 seconds ago HBASE-4785 Improve recovery time of HBase client when a region server dies. diff --git src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java index f317524..e4de22a 100644 --- src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -23,7 +23,9 @@ import java.io.Closeable; import java.io.IOException; import java.lang.reflect.Proxy; import java.lang.reflect.UndeclaredThrowableException; +import java.net.ConnectException; import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -31,8 +33,8 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Map.Entry; +import java.util.NoSuchElementException; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Callable; @@ -512,6 +514,13 @@ public class HConnectionManager { cachedRegionLocations = new HashMap>(); + // The presence of a server in the map implies it's likely that there is an + // entry in cachedRegionLocations that map to this server; but the absence + // of a server in this map guarentees that there is no entry in cache that + // maps to the absent server. + private final Set cachedServers = + new HashSet(); + // region cache prefetch is enabled by default. this set contains all // tables whose region cache prefetch are disabled. private final Set regionCachePrefetchDisabledTables = @@ -1117,6 +1126,35 @@ public class HConnectionManager { } /* + * Delete all cached entries of a table that maps to a specific location. + * + * @param tablename + * @param server + */ + private void clearCachedLocationForServer( + final String server) { + boolean deletedSomething = false; + synchronized (this.cachedRegionLocations) { + if (!cachedServers.contains(server)) { + return; + } + for (SoftValueSortedMap tableLocations : + cachedRegionLocations.values()) { + for (Entry e : tableLocations.entrySet()) { + if (e.getValue().getServerAddress().toString().equals(server)) { + tableLocations.remove(e.getKey()); + deletedSomething = true; + } + } + } + cachedServers.remove(server); + } + if (deletedSomething && LOG.isDebugEnabled()) { + LOG.debug("Removed all cached region locations that map to " + server); + } + } + + /* * @param tableName * @return Map of cached locations for passed tableName */ @@ -1141,6 +1179,7 @@ public class HConnectionManager { public void clearRegionCache() { synchronized(this.cachedRegionLocations) { this.cachedRegionLocations.clear(); + this.cachedServers.clear(); } } @@ -1159,7 +1198,12 @@ public class HConnectionManager { byte [] startKey = location.getRegionInfo().getStartKey(); SoftValueSortedMap tableLocations = getTableLocations(tableName); - if (tableLocations.put(startKey, location) == null) { + boolean hasNewCache = false; + synchronized (this.cachedRegionLocations) { + cachedServers.add(location.getServerAddress().toString()); + hasNewCache = (tableLocations.put(startKey, location) == null); + } + if (hasNewCache) { LOG.debug("Cached location for " + location.getRegionInfo().getRegionNameAsString() + " is " + location.getHostnamePort()); @@ -1282,6 +1326,17 @@ public class HConnectionManager { } catch (Throwable t) { callable.shouldRetry(t); t = translateException(t); + if (t instanceof SocketTimeoutException || + t instanceof ConnectException || + t instanceof RetriesExhaustedException) { + // if thrown these exceptions, we clear all the cache entries that + // map to that slow/dead server; otherwise, let cache miss and ask + // .META. again to find the new location + HRegionLocation hrl = callable.location; + if (hrl != null) { + clearCachedLocationForServer(hrl.getServerAddress().toString()); + } + } RetriesExhaustedException.ThrowableWithExtraContext qt = new RetriesExhaustedException.ThrowableWithExtraContext(t, System.currentTimeMillis(), callable.toString()); diff --git src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java index 4d1a552..11dfbef 100644 --- src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java +++ src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java @@ -24,6 +24,7 @@ import java.lang.ref.SoftReference; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -172,7 +173,13 @@ public class SoftValueSortedMap implements SortedMap { } public synchronized Set> entrySet() { - throw new RuntimeException("Not implemented"); + checkReferences(); + Set>> entries = this.internalMap.entrySet(); + Set> realEntries = new LinkedHashSet>(); + for (Map.Entry> entry : entries) { + realEntries.add(entry.getValue()); + } + return realEntries; } public synchronized Collection values() { @@ -185,12 +192,24 @@ public class SoftValueSortedMap implements SortedMap { return hardValues; } - private static class SoftValue extends SoftReference { + private static class SoftValue extends SoftReference implements Map.Entry { final K key; SoftValue(K key, V value, ReferenceQueue q) { super(value, q); this.key = key; } + + public K getKey() { + return this.key; + } + + public V getValue() { + return get(); + } + + public V setValue(V value) { + throw new RuntimeException("Not implemented"); + } } }