Index: src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (revision ) +++ src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (working copy) @@ -175,7 +175,7 @@ public abstract class EventHandler implements Runnable, Comparable { process(); if (getListener() != null) getListener().afterProcess(this); } catch(Throwable t) { - LOG.error("Caught throwable while processing event " + eventType, t); + handleException(t); } } @@ -262,4 +262,12 @@ public abstract class EventHandler implements Runnable, Comparable { public String getInformativeName() { return this.getClass().toString(); } + + /** + * Event exception handler, may be overridden + * @param t Throwable object + */ + protected void handleException(Throwable t) { + LOG.error("Caught throwable while processing event " + eventType, t); + } } Index: src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java (revision ) +++ src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java (working copy) @@ -20,7 +20,9 @@ package org.apache.hadoop.hbase.master.handler; import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HRegionInfo; @@ -28,6 +30,7 @@ import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.master.DeadServer; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.util.Threads; import org.apache.zookeeper.KeeperException; /** @@ -38,6 +41,10 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler { private final boolean carryingRoot; private final boolean carryingMeta; private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class); + private AtomicInteger eventExceptionCount = new AtomicInteger(0); + @VisibleForTesting + static final int SHOW_STRACKTRACE_FREQUENCY = 100; + public MetaServerShutdownHandler(final Server server, final MasterServices services, final DeadServer deadServers, final ServerName serverName, @@ -116,6 +123,8 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler { } super.process(); + // Clear this counter on successful handling. + this.eventExceptionCount.set(0); } /** * Before assign the ROOT region, ensure it haven't @@ -196,4 +205,20 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler { } return getClass().getSimpleName() + "-" + name + "-" + getSeqid(); } + + @Override + protected void handleException(Throwable t) { + int count = eventExceptionCount.getAndIncrement(); + if (count < 0) count = eventExceptionCount.getAndSet(0); + if (count > SHOW_STRACKTRACE_FREQUENCY) { // Too frequent, let's slow reporting + Threads.sleep(1000); + } + if (count % SHOW_STRACKTRACE_FREQUENCY == 0) { + LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount, t); + } else { + LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount + + "; " + t.getMessage() + "; stack trace shows every " + SHOW_STRACKTRACE_FREQUENCY + + "th time."); + } + } }