diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterRestartWithProcedures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterRestartWithProcedures.java new file mode 100644 index 0000000..2bff447 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterRestartWithProcedures.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.procedure; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Callable; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@Category(LargeTests.class) +public class TestMasterRestartWithProcedures { + private static final Log LOG = LogFactory.getLog(TestMasterFailoverWithProcedures.class); + + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static void setupConf(Configuration conf) { + conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 1); + } + + @Before + public void setup() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(1, 1); + } + + @After + public void tearDown() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + @Test(timeout=180000) + public void testMasterProcsWithMasterRestarting() throws Exception { + final TableName tableName = TableName.valueOf("testMasterProcsWithMasterRestarting"); + final int restartInterval = 7000; + final int maxOpRetries = 10; + final int numRuns = 7; + + AtomicBoolean running = new AtomicBoolean(true); + Thread masterRestartThread = initRestartMasterAtIntervalThread(3500, restartInterval, running); + masterRestartThread.setName("MasterRestartThread"); + masterRestartThread.start(); + final AtomicInteger count = new AtomicInteger(0); + try { + final Admin admin = UTIL.getHBaseAdmin(); + for (; count.get() < numRuns && running.get(); count.incrementAndGet()) { + long startTime = System.currentTimeMillis(); + + // create the table + final byte[][] splitKeys = new byte[][] { + Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") + }; + final HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("f1"); + htd.addFamily(hcd); + runRollbackableOperation(maxOpRetries, new Callable() { + @Override + public Void call() throws Exception { + LOG.info("executing create table step=" + count.get()); + try { + admin.createTable(htd, splitKeys); + } catch (TableExistsException e) { + LOG.warn("TODO HBASE-13415 Use nonces for double submits from client: " + + e.getMessage()); + } + return null; + } + }); + + // disable the table + runRollbackableOperation(maxOpRetries, new Callable() { + @Override + public Void call() throws Exception { + LOG.info("executing disable table step=" + count.get()); + admin.disableTable(tableName); + return null; + } + }); + + // enable the table + runRollbackableOperation(maxOpRetries, new Callable() { + @Override + public Void call() throws Exception { + LOG.info("executing enable table step=" + count.get()); + admin.enableTable(tableName); + return null; + } + }); + + // disable the table + runRollbackableOperation(maxOpRetries, new Callable() { + @Override + public Void call() throws Exception { + LOG.info("executing disable table step=" + count.get()); + admin.disableTable(tableName); + return null; + } + }); + + // delete the table + LOG.info("executing delete table step=" + count.get()); + try { + admin.deleteTable(tableName); + } catch (TableNotFoundException e) { + LOG.warn("TODO HBASE-13415 Use nonces for double submits from client: " + e.getMessage()); + } + + LOG.debug("executing step=" + count.get() + + " took " + String.format("%.3fsec", (System.currentTimeMillis() - startTime) / 1000.0f)); + } + } catch (Throwable e) { + LOG.error("master operation failed: " + e.getMessage(), e); + throw e; + } finally { + running.set(false); + masterRestartThread.join(); + } + assertEquals(numRuns, count.get()); + } + + private T runRollbackableOperation(int maxRetries, final Callable callable) + throws Exception { + Exception exception = null; + for (int i = 0; i < maxRetries; ++i) { + try { + return callable.call(); + } catch (Exception e) { + LOG.warn("Operation failed attempt=" + i + ": " + e.getMessage(), e); + exception = e; + } + } + throw exception; + } + + private Thread initRestartMasterAtIntervalThread(final int minInterval, final int maxInterval, + final AtomicBoolean running) { + return new Thread() { + @Override + public void run() { + for (int i = 0; running.get(); ++i) { + try { + long sleepInterval = Math.max(minInterval, Math.round(maxInterval * Math.random())); + LOG.debug("Sleeping for " + sleepInterval + + " before executing next master restart count=" + i); + Threads.sleepWithoutInterrupt(sleepInterval); + if (!running.get()) break; + + while (UTIL.getHBaseCluster().getMaster() == null) { + Threads.sleepWithoutInterrupt(250); + } + ServerName serverName = UTIL.getHBaseCluster().getMaster().getServerName(); + LOG.info("executing master kill"); + UTIL.getHBaseCluster().killMaster(serverName); + UTIL.getHBaseCluster().waitForMasterToStop(serverName, 10000); + LOG.info("executing master restart"); + UTIL.getHBaseCluster().startMaster(); + UTIL.getHBaseCluster().waitForActiveAndReadyMaster(10000); + } catch (IOException e) { + LOG.error("master restart failed: " + e.getMessage(), e); + running.set(false); + } catch (Throwable e) { + LOG.warn("executing unhandled exception " + e.getMessage(), e); + } + } + } + }; + } +}