diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ChoreService.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ChoreService.java index 85d6131b50..5017432ade 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ChoreService.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ChoreService.java @@ -191,6 +191,11 @@ public class ChoreService implements ChoreServicer { ScheduledFuture future = scheduledChores.get(chore); future.cancel(mayInterruptIfRunning); scheduledChores.remove(chore); + try { + chore.cleanup(); + } catch (Throwable t) { + LOG.error(chore + " cleanup failed; ignoring", t); + } // Removing a chore that was missing its start time means it may be possible // to reduce the number of threads diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java index 468b5d30c3..61bfe6ef03 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java @@ -70,6 +70,7 @@ public abstract class ScheduledChore implements Runnable { private long timeOfLastRun = -1; // system time millis private long timeOfThisRun = -1; // system time millis private boolean initialChoreComplete = false; + private boolean cleanupExecuted = false; /** * A means by which a ScheduledChore can be stopped. Once a chore recognizes that it has been @@ -348,10 +349,18 @@ public abstract class ScheduledChore implements Runnable { return true; } + public final synchronized void cleanup() { + if (this.cleanupExecuted) { + return; + } + this.cleanupExecuted = true; + cleanupImpl(); + } + /** * Override to run cleanup tasks when the Chore encounters an error and must stop running */ - protected synchronized void cleanup() { + protected void cleanupImpl() { } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java index af35ce4eb3..4aae01b818 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java @@ -165,7 +165,8 @@ public class ClusterStatusPublisher extends ScheduledChore { } @Override - protected synchronized void cleanup() { + protected void cleanupImpl() { + super.cleanupImpl(); connected = false; publisher.close(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 5a449a047c..090be83a33 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -795,7 +795,7 @@ public class HMaster extends HRegionServer implements MasterServices { if (clientQuorumServers != null && !clientZkObserverMode) { // we need to take care of the ZK information synchronization // if given client ZK are not observer nodes - ZKWatcher clientZkWatcher = new ZKWatcher(conf, + this.clientZkWatcher = new ZKWatcher(conf, getProcessName() + ":" + rpcServices.getSocketAddress().getPort() + "-clientZK", this, false, true); this.metaLocationSyncer = new MetaLocationSyncer(zooKeeper, clientZkWatcher, this); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java index 6c5d677a86..f8c5b294a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java @@ -89,8 +89,8 @@ public class MobCompactionChore extends ScheduledChore { } @Override - protected synchronized void cleanup() { - super.cleanup(); + protected void cleanupImpl() { + super.cleanupImpl(); pool.shutdown(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java index 19a7a693ef..305881dc78 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java @@ -435,7 +435,8 @@ public abstract class CleanerChore extends Schedu } @Override - public synchronized void cleanup() { + public void cleanupImpl() { + super.cleanupImpl(); for (T lc : this.cleanersChain) { try { lc.stop("Exiting"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java index 7ad6177764..0198f67f3a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java @@ -203,8 +203,8 @@ public class HFileCleaner extends CleanerChore { } @Override - public synchronized void cleanup() { - super.cleanup(); + public void cleanupImpl() { + super.cleanupImpl(); stopHFileDeleteThreads(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java index a7338c0698..f2409fbbfe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java @@ -123,8 +123,8 @@ public class LogCleaner extends CleanerChore { } @Override - public synchronized void cleanup() { - super.cleanup(); + public void cleanupImpl() { + super.cleanupImpl(); interruptOldWALsCleaner(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index f983882b6f..15387bc222 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -398,6 +398,7 @@ public class HRegionServer extends HasThread implements // zookeeper connection and watcher protected final ZKWatcher zooKeeper; + protected ZKWatcher clientZkWatcher; // master address tracker private final MasterAddressTracker masterAddressTracker; @@ -1184,6 +1185,9 @@ public class HRegionServer extends HasThread implements if (this.zooKeeper != null) { this.zooKeeper.close(); } + if (this.clientZkWatcher != null) { + this.clientZkWatcher.close(); + } this.shutDown = true; LOG.info("Exiting; stopping=" + this.serverName + "; zookeeper connection closed."); }