Index: hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java (revision 1563959) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java (working copy) @@ -429,11 +429,14 @@ callable.setClose(); try { this.caller.callWithRetries(callable); + } catch (UnknownScannerException e) { + // We used to catch this error, interpret, and rethrow. However, we + // have since decided that it's not nice for a scanner's close to + // throw exceptions. Chances are it was just an UnknownScanner + // exception due to lease time out. } catch (IOException e) { - // We used to catch this error, interpret, and rethrow. However, we - // have since decided that it's not nice for a scanner's close to - // throw exceptions. Chances are it was just an UnknownScanner - // exception due to lease time out. + /* An exception other than UnknownScanner is unexpected. */ + LOG.warn("scanner failed to close. Exception follows: " + e); } callable = null; } Index: hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java (revision 1563959) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java (working copy) @@ -160,7 +160,9 @@ try { tr = new TimeRange(timestamp, timestamp+1); } catch(IOException e) { - // Will never happen + // This should never happen, unless integer overflow or something extremely wrong... + throw new RuntimeException("TimeRange failed, likely caused by integer overflow. " + + "Exception thrown to prevent bad things from propagating. ", e); } return this; } Index: hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java (revision 1563959) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java (working copy) @@ -301,7 +301,9 @@ try { tr = new TimeRange(timestamp, timestamp+1); } catch(IOException e) { - // Will never happen + // This should never happen, unless integer overflow or something extremely wrong. + throw new RuntimeException("TimeRange failed, likely caused by integer overflow. " + + "Exception thrown to prevent bad things from propagating. ", e); } return this; } Index: hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java =================================================================== --- hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java (revision 1563959) +++ hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java (working copy) @@ -145,7 +145,12 @@ if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) { isShowConf = true; } - } catch (Exception e) { + } catch (LinkageError e) { + // should we handle it more aggressively in addition to log the error? + LOG.warn("LinkageError thrown: " + e); + } catch (ClassNotFoundException ce) { + LOG.debug("ClassNotFound: ConfServlet"); + // ignore } return isShowConf; } Index: hbase-common/src/main/java/org/apache/hadoop/hbase/util/JVM.java =================================================================== --- hbase-common/src/main/java/org/apache/hadoop/hbase/util/JVM.java (revision 1563959) +++ hbase-common/src/main/java/org/apache/hadoop/hbase/util/JVM.java (working copy) @@ -207,7 +207,8 @@ if (input != null){ try { input.close(); - } catch (IOException ignored) { + } catch (IOException e) { + LOG.warn("Not able to close the InputStream", e); } } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java (revision 1563959) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java (working copy) @@ -62,7 +62,11 @@ try { prefixLength = Integer.parseInt(prefixLengthString); } catch (NumberFormatException nfe) { - // ignore + /* Differentiate NumberFormatException from an invalid value range reported below. */ + LOG.error("Number format exception when parsing " + PREFIX_LENGTH_KEY + " for table " + + region.getTableDesc().getTableName() + ":" + + prefixLengthString + ". " + nfe); + return; } if (prefixLength <= 0) { LOG.error("Invalid value for " + PREFIX_LENGTH_KEY + " for table " Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java (revision 1563959) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java (working copy) @@ -128,12 +128,19 @@ protected void releaseTableLock() { if (this.tableLock != null) { - try { - this.tableLock.release(); - } catch (IOException ex) { - LOG.warn("Could not release the table lock", ex); - //TODO: if we get here, and not abort RS, this lock will never be released + for (int i = 1; i < 4; ++i) { // we retry at most 3 times + try { + this.tableLock.release(); + return; // success! + } catch (IOException ex) { + LOG.warn("Failed to release tableLock on " + region_a.getRegionNameAsString() + + " on try #" + i + ", exception follows: " + ex); + } } + LOG.error("Could not release the table lock. " + + "Aborting this server to avoid holding the lock forever."); + this.server.abort("Abort; we got an error when releasing the table lock " + + "on " + region_a.getRegionNameAsString()); } } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java (revision 1563959) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java (working copy) @@ -129,12 +129,19 @@ protected void releaseTableLock() { if (this.tableLock != null) { - try { - this.tableLock.release(); - } catch (IOException ex) { - LOG.warn("Could not release the table lock", ex); - //TODO: if we get here, and not abort RS, this lock will never be released - } + for (int i = 1; i < 4; ++i) { // we retry at most 3 times + try { + this.tableLock.release(); + return; // success! + } catch (IOException ex) { + LOG.warn("Failed to release tableLock on " + parent.getRegionNameAsString() + + " on try #" + i + ", exception follows: " + ex); + } + } + LOG.error("Could not release the table lock. " + + "Aborting this server to avoid holding the lock forever."); + this.server.abort("Abort; we got an error when releasing the table lock " + + "on " + parent.getRegionNameAsString()); } } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java (revision 1563959) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java (working copy) @@ -255,8 +255,15 @@ Field fEnd = SequenceFile.Reader.class.getDeclaredField("end"); fEnd.setAccessible(true); end = fEnd.getLong(this.reader); - } catch(Exception e) { /* reflection fail. keep going */ } - + } catch(NoSuchFieldException nfe) { + /* reflection failure, keep going */ + } catch(IllegalAccessException iae) { + /* reflection failure, keep going */ + } catch(Exception e) { + /* All other cases. Should we handle it more aggressively? */ + LOG.warn("Unexpected exception when accessing the end field", e); + } + String msg = (this.path == null? "": this.path.toString()) + ", entryStart=" + entryStart + ", pos=" + pos + ((end == Long.MAX_VALUE) ? "" : ", end=" + end) + @@ -268,8 +275,14 @@ .getConstructor(String.class) .newInstance(msg) .initCause(ioe); - } catch(Exception e) { /* reflection fail. keep going */ } - + } catch(NoSuchMethodException nfe) { + /* reflection failure, keep going */ + } catch(IllegalAccessException iae) { + /* reflection failure, keep going */ + } catch(Exception e) { + /* All other cases. Should we handle it more aggressively? */ + LOG.warn("Unexpected exception when accessing the end field", e); + } return ioe; } }