From 0e0e1938464ff6310d4569db834b1ad790b0c89e Mon Sep 17 00:00:00 2001 From: chenheng Date: Sat, 24 Oct 2015 13:20:18 +0800 Subject: [PATCH] HBASE-14680 Two configs for snapshot timeout and better defaults --- hbase-common/src/main/resources/hbase-default.xml | 7 ++++++ .../snapshot/DisabledTableSnapshotHandler.java | 25 ---------------------- .../hbase/master/snapshot/SnapshotManager.java | 14 ++++-------- .../hbase/snapshot/SnapshotDescriptionUtils.java | 20 ++++++++++++++--- 4 files changed, 28 insertions(+), 38 deletions(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 1654391..10a557b 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -1686,4 +1686,11 @@ possible configurations would overwhelm and obscure the important. The max number of threads used in MobCompactor. + + hbase.snapshot.master.timeout.millis + 300000 + + timeout for the snapshot procedure execution + + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java index 1f2ed0a..c38971b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java @@ -56,7 +56,6 @@ import org.apache.zookeeper.KeeperException; @InterfaceStability.Evolving public class DisabledTableSnapshotHandler extends TakeSnapshotHandler { private static final Log LOG = LogFactory.getLog(DisabledTableSnapshotHandler.class); - private final TimeoutExceptionInjector timeoutInjector; /** * @param snapshot descriptor of the snapshot to take @@ -65,9 +64,6 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler { public DisabledTableSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices) { super(snapshot, masterServices); - - // setup the timer - timeoutInjector = getMasterTimerAndBindToMonitor(snapshot, conf, monitor); } @Override @@ -81,8 +77,6 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler { public void snapshotRegions(List> regionsAndLocations) throws IOException, KeeperException { try { - timeoutInjector.start(); - // 1. get all the regions hosting this table. // extract each pair to separate lists @@ -127,25 +121,6 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler { } finally { LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot) + " as finished."); - - // 3. mark the timer as finished - even if we got an exception, we don't need to time the - // operation any further - timeoutInjector.complete(); } } - - - /** - * Create a snapshot timer for the master which notifies the monitor when an error occurs - * @param snapshot snapshot to monitor - * @param conf configuration to use when getting the max snapshot life - * @param monitor monitor to notify when the snapshot life expires - * @return the timer to use update to signal the start and end of the snapshot - */ - private TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot, - Configuration conf, ForeignExceptionListener monitor) { - long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(), - SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); - return new TimeoutExceptionInjector(monitor, maxTime); - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index 38aa2c0..76a2fd9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -122,15 +122,6 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable */ private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis"; - /** By default, check to see if the snapshot is complete (ms) */ - private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000; - - /** - * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for - * completion. - */ - private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis"; - /** Name of the operation to use in the controller */ public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot"; @@ -1078,7 +1069,10 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable // get the configuration for the coordinator Configuration conf = master.getConfiguration(); long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT); - long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT); + long timeoutMillis = Math.max(conf.getLong(SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_KEY, + SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT), + conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS, + SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME)); int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT); // setup the default procedure coordinator diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java index 2fc5d83..60a251e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java @@ -105,10 +105,23 @@ public class SnapshotDescriptionUtils { /** Default value if no start time is specified */ public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0; + public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis"; - /** By default, wait 60 seconds for a snapshot to complete */ - public static final long DEFAULT_MAX_WAIT_TIME = 60000; + /** By default, wait 300 seconds for a snapshot to complete */ + public static final long DEFAULT_MAX_WAIT_TIME = 60000 * 5 ; + + + /** By default, check to see if the snapshot is complete (ms) */ + @Deprecated + public static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000 * 5; + + /** + * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for + * completion. + */ + @Deprecated + public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis"; private SnapshotDescriptionUtils() { // private constructor for utility class @@ -128,7 +141,8 @@ public class SnapshotDescriptionUtils { default: confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS; } - return conf.getLong(confKey, defaultMaxWaitTime); + return Math.max(conf.getLong(confKey, defaultMaxWaitTime), + conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, defaultMaxWaitTime)); } /** -- 1.9.3 (Apple Git-50)