diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index bca819e..90bf5a5 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -117,7 +117,7 @@ import org.apache.zookeeper.Watcher.Event.KeeperState; * the HMaster. There are many HRegionServers in a single HBase deployment. */ public class HRegionServer implements HRegionInterface, - HBaseRPCErrorHandler, Runnable, Watcher { + HBaseRPCErrorHandler, Runnable, Watcher, Stop { public static final Log LOG = LogFactory.getLog(HRegionServer.class); private static final HMsg REPORT_EXITING = new HMsg(Type.MSG_REPORT_EXITING); private static final HMsg REPORT_QUIESCED = new HMsg(Type.MSG_REPORT_QUIESCED); @@ -2492,5 +2492,4 @@ public class HRegionServer implements HRegionInterface, HRegionServer.class); doMain(args, regionServerClass); } - -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHookManager.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHookManager.java new file mode 100644 index 0000000..bc6a7d4 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ShutdownHookManager.java @@ -0,0 +1,170 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.lang.reflect.Field; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.util.Threads; + +/** + * Manages the hdfs and regionserver shutdown hooks. Suppress shutdown hooks + * by setting {@link ShutdownHookManager#RUN_SHUTDOWN_HOOK} in the Configuration + * to false. + */ +class ShutdownHookManager { + private static final Log LOG = LogFactory.getLog(ShutdownHookManager.class); + + /** + * Key for boolean configuration whose default is true. + */ + public static final String RUN_SHUTDOWN_HOOK = "hbase.shutdown.hook"; + + /** + * Key for a long configuration on how much time to wait on the fs shutdown + * hook. Default is 30 seconds. + */ + public static final String FS_SHUTDOWN_HOOK_WAIT = "hbase.fs.shutdown.hook.wait"; + + ShutdownHookManager(final Configuration conf, final Stop stop, + final Thread threadToJoin) { + Thread fsShutdownHook = suppressHdfsShutdownHook(); + Thread t = new ShutdownHookThread(conf, stop, threadToJoin, fsShutdownHook); + Runtime.getRuntime().addShutdownHook(t); + LOG.info("Installed shutdown hook thread: " + t.getName()); + } + + /* + * Thread to shutdown the region server in an orderly manner. This thread + * is registered as a shutdown hook in the HRegionServer constructor and is + * only called when the HRegionServer receives a kill signal. + */ + private static class ShutdownHookThread extends Thread { + private final Stop stop; + private final Thread threadToJoin; + private final Thread fsShutdownHook; + private final Configuration conf; + + + /** + * Shutdown hook thread. + * @param conf + * @param stop Instance to call stop on when shutdown hook runs. + * @param threadToJoin After calling {@link Lifecycle#stop()}, we'll join + * on this thread till its dead. + * @param fsShutdownHook + */ + public ShutdownHookThread(final Configuration conf, final Stop stop, + final Thread threadToJoin, final Thread fsShutdownHook) { + super("Shutdownhook:" + threadToJoin.getName()); + this.stop = stop; + this.threadToJoin = threadToJoin; + this.conf = conf; + this.fsShutdownHook = fsShutdownHook; + } + + @Override + public void run() { + LOG.info("Starting shutdown hook thread."); + // Tell service to stop. + this.stop.stop(); + // Wait for main thread to exit. + Threads.shutdown(this.threadToJoin); + if (this.fsShutdownHook == null || + this.conf.getBoolean(RUN_SHUTDOWN_HOOK, true)) { + LOG.info("Starting fs shutdown hook thread."); + this.fsShutdownHook.start(); + Threads.shutdown(this.fsShutdownHook, + this.conf.getLong(FS_SHUTDOWN_HOOK_WAIT, 30000)); + } + LOG.info("Finished shutdown hook thread."); + } + } + + /** + * So, HDFS keeps a static map of all FS instances. In order to make sure + * things are cleaned up on our way out, it also creates a shutdown hook + * so that all filesystems can be closed when the process is terminated; it + * calls FileSystem.closeAll. This inconveniently runs concurrently with our + * own shutdown handler, and therefore causes all the filesystems to be closed + * before the server can do all its necessary cleanup. + * + *
The dirty reflection in this method sneaks into the FileSystem cache + * and grabs the shutdown hook, removes it from the list of active shutdown + * hooks, and hangs onto it until later. Then, after we're properly done with + * our graceful shutdown, we can execute the hdfs hook manually to make sure + * loose ends are tied up. + * + *
This seems quite fragile and susceptible to breaking if Hadoop changes + * anything about the way this cleanup is managed. Keep an eye on things. + * @return The fs shutdown hook or null if we failed to remove the shutdown + * hook. + */ + private Thread suppressHdfsShutdownHook() { + try { + Field field = FileSystem.class.getDeclaredField ("clientFinalizer"); + field.setAccessible(true); + Thread hdfsClientFinalizer = (Thread)field.get(null); + if (hdfsClientFinalizer == null) { + throw new RuntimeException("client finalizer is null, can't suppress!"); + } + if (!Runtime.getRuntime().removeShutdownHook(hdfsClientFinalizer)) { + LOG.error("Failed suppresion of fs shutdown hook: " + hdfsClientFinalizer); + return null; + } + return hdfsClientFinalizer; + } catch (NoSuchFieldException nsfe) { + LOG.fatal("Couldn't find field 'clientFinalizer' in FileSystem!", nsfe); + throw new RuntimeException("Failed to suppress HDFS shutdown hook"); + } catch (IllegalAccessException iae) { + LOG.fatal("Couldn't access field 'clientFinalizer' in FileSystem!", iae); + throw new RuntimeException("Failed to suppress HDFS shutdown hook"); + } + } + + /** + * Main to test basic functionality. + * @param args + * @throws IOException + */ + public static void main(final String [] args) throws IOException { + Configuration conf = HBaseConfiguration.create(); + // Instantiate a FileSystem. This will register the fs shutdown hook. + FileSystem.get(conf); + Thread donothing = new Thread("donothing") { + @Override + public void run() { + super.run(); + } + }; + donothing.start(); + new ShutdownHookManager(conf, new Stop() { + @Override + public void stop() { + // Nothing to do. + } + }, donothing); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/Stop.java b/src/main/java/org/apache/hadoop/hbase/regionserver/Stop.java new file mode 100644 index 0000000..3fffb6d --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/Stop.java @@ -0,0 +1,36 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +/** + * Implementations have a {@link #stop()}. + */ +interface Stop { + // Starting small, just doing a stop for now. Doing Lifecycle seemed a + // stretch especially when a RegionServer is hosted in a Thread (can't do + // 'stop' on a Thread and 'start' has special meaning for Threads) and then + // Master is implemented differently again (it is a Thread itself). We + // should move to redoing Master and RegionServer servers to use Spring or + // exploit some other container but for now, I just need stop. + /** + * Stop service. + */ + public void stop(); +}