diff --git bin/hbase bin/hbase index 4eeb145..4674693 100755 --- bin/hbase +++ bin/hbase @@ -355,7 +355,7 @@ fi # Exec unless HBASE_NOEXEC is set. if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" + "$JAVA" -Dproc_$COMMAND {-XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" else - exec "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" + exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" fi diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java new file mode 100644 index 0000000..af54b82 --- /dev/null +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.Collection; +import java.util.Random; + +import junit.framework.Assert; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.LoadTestTool; +import org.apache.hadoop.hbase.util.SimpleStoppable; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * A system test which does large data ingestion, while killing the region servers + * and the master(s) randomly, and does a verify afterwards. + */ +@Category(IntegrationTests.class) +public class IntegrationTestDataIngestWithChaosMonkey { + + private static final Log LOG = LogFactory.getLog(IntegrationTestDataIngestWithChaosMonkey.class); + private HBaseTestingUtility util; + private HBaseCluster cluster; + private static final String TABLE_NAME = "IntegrationTestDataIngestWithChaosMonkey"; + private static final int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster + + @Before + public void setUp() throws Exception { + util = new HBaseTestingUtility(); + if (!util.isRealCluster()) { + util.startMiniCluster(NUM_SLAVES_BASE); + } + + cluster = util.getHBaseClusterShim(); + deleteTableIfNecessary(); + } + + @After + public void tearDown() throws Exception { + if (!util.isRealCluster()) { + util.shutdownMiniCluster(); + } + } + + private void deleteTableIfNecessary() { + try { + util.deleteTable(Bytes.toBytes(TABLE_NAME)); + } catch (Exception ex) { + //ignore + } + } + + @Test + public void testDataIngest() throws Exception { + LOG.info("Running testDataIngest"); + SimpleChaosMonkey monkey = new SimpleChaosMonkey(util); + Thread monkeyThread = new Thread(monkey); + + monkeyThread.start(); + + LoadTestTool loadTool = new LoadTestTool(); + loadTool.setConf(util.getConfiguration()); + + String numKeys = String.valueOf(estimateDataSize()); + + int ret = loadTool.run(new String[] { + "-tn", TABLE_NAME, + "-write", "10:100:20", + "-num_keys", numKeys + }); + + monkey.stop("test has finished, that's why"); + monkeyThread.join(); + + //assert that load was successful + Assert.assertEquals(0, ret); + + ret = loadTool.run(new String[] { + "-tn", TABLE_NAME, + "-read", "100:20", + "-num_keys", numKeys + }); + + //assert that verify was successful + Assert.assertEquals(0, ret); + } + + /** Estimates a data size based on the cluster size */ + protected long estimateDataSize() throws IOException { + //base is a 4 slave node cluster. + ClusterStatus status = cluster.getClusterStatus(); + int numRegionServers = status.getServersSize(); + int multiplication = Math.max(1, numRegionServers / NUM_SLAVES_BASE); + + return 100000 * multiplication; + } + + /** + * A very simple utility which randomly selects a region server, kill's it, + * and restarts it after some time. + * + * @see http://www.codinghorror.com/blog/2011/04/working-with-the-chaos-monkey.html + */ + static class SimpleChaosMonkey extends SimpleStoppable implements Runnable { + private static final long ONE_MIN = 60000; + private static final long TIMEOUT = ONE_MIN; + + HBaseTestingUtility util; + + public SimpleChaosMonkey(HBaseTestingUtility util) { + this.util = util; + } + + @Override + public void run() { + try { + HBaseCluster cluster = util.getHBaseClusterShim(); + ClusterStatus status = cluster.getInitialClusterStatus(); + Collection regionServers = status.getServers(); + ServerName[] servers = regionServers.toArray(new ServerName[regionServers.size()]); + + int numRegionServers = status.getServersSize(); + Random random = new Random(); + + while (!isStopped()) { + int selected = random.nextInt(numRegionServers); + Assert.assertTrue(cluster.abortRegionServer(servers[selected])); + cluster.waitForRegionServerToStop(servers[selected], TIMEOUT); + sleep(ONE_MIN); + + Assert.assertTrue(cluster.startRegionServer(servers[selected].getHostname())); + cluster.waitForRegionServerToStart(servers[selected].getHostname(), TIMEOUT); + sleep(ONE_MIN); + } + } catch(IOException ex) { + throw new RuntimeException(ex); + } + } + + void sleep(long milis) { + try { + Thread.sleep(milis); + } catch (InterruptedException ex) { + //ignore + } + } + } +} diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java new file mode 100644 index 0000000..ac85c40 --- /dev/null +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestsDriver.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.hadoop.hbase.util.AbstractHBaseTool; +import org.apache.hadoop.util.ToolRunner; +import org.junit.internal.TextListener; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; + +/** + * This class drives the Integration test suite execution. Executes all + * tests having @Category(IntegrationTests.class) annotation. + */ +public class IntegrationTestsDriver extends AbstractHBaseTool { + + public static void main(String[] args) throws Exception { + int ret = ToolRunner.run(new IntegrationTestsDriver(), args); + System.exit(ret); + } + + @Override + protected void addOptions() { + } + + @Override + protected void processOptions(CommandLine cmd) { + } + + /** + * Returns test classes annotated with @Category(IntegrationTests.class) + */ + private Class[] findIntegrationTestClasses() throws ClassNotFoundException, IOException { + TestCheckTestClasses util = new TestCheckTestClasses(); + List> classes = util.findTestClasses(IntegrationTests.class); + return classes.toArray(new Class[classes.size()]); + } + + @Override + protected int doWork() throws Exception { + + //this is called from the command line, so we should set to use the real cluster + HBaseTestingUtility.setUseRealCluster(conf); + + JUnitCore junit = new JUnitCore(); + junit.addListener(new TextListener(System.out)); + Result result = junit.run(findIntegrationTestClasses()); + + return result.wasSuccessful() ? 0 : 1; + } + +} \ No newline at end of file diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java index ab53085..e3633c9 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java @@ -65,7 +65,7 @@ public abstract class AbstractHBaseTool implements Tool { protected abstract void processOptions(CommandLine cmd); /** The "main function" of the tool */ - protected abstract void doWork() throws Exception; + protected abstract int doWork() throws Exception; @Override public Configuration getConf() { @@ -101,13 +101,14 @@ public abstract class AbstractHBaseTool implements Tool { processOptions(cmd); + int ret = EXIT_FAILURE; try { - doWork(); + ret = doWork(); } catch (Exception e) { LOG.error("Error running command-line tool", e); return EXIT_FAILURE; } - return EXIT_SUCCESS; + return ret; } private boolean sanityCheckOptions(CommandLine cmd) { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/ClusterManager.java hbase-server/src/test/java/org/apache/hadoop/hbase/ClusterManager.java new file mode 100644 index 0000000..8ff2d7e --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/ClusterManager.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configured; + + +/** + * ClusterManager is an api to manage servers in a distributed environment. It provides services + * for starting / stopping / killing Hadoop/HBase daemons. Concrete implementations provide actual + * functionality for carrying out deployment-specific tasks. + */ +@InterfaceAudience.Private +public abstract class ClusterManager extends Configured { + protected static final Log LOG = LogFactory.getLog(ClusterManager.class); + + private static final String SIGKILL = "SIGKILL"; + private static final String SIGSTOP = "SIGSTOP"; + private static final String SIGCONT = "SIGCONT"; + + public ClusterManager() { + } + + /** + * Type of the service daemon + */ + public static enum ServiceType { + HADOOP_NAMENODE("namenode"), + HADOOP_DATANODE("datanode"), + HADOOP_JOBTRACKER("jobtracker"), + HADOOP_TASKTRACKER("tasktracker"), + HBASE_MASTER("master"), + HBASE_REGIONSERVER("regionserver"); + + private String name; + + ServiceType(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return getName(); + } + } + + /** + * Start the service on the given host + */ + public abstract boolean start(ServiceType service, String hostname) throws IOException; + + /** + * Stop the service on the given host + */ + public abstract boolean stop(ServiceType service, String hostname) throws IOException; + + /** + * Restart the service on the given host + */ + public abstract boolean restart(ServiceType service, String hostname) throws IOException; + + /** + * Send the given posix signal to the service + */ + public abstract boolean signal(ServiceType service, String signal, + String hostname) throws IOException; + + /** + * Kill the service running on given host + */ + public boolean kill(ServiceType service, String hostname) throws IOException { + return signal(service, SIGKILL, hostname); + } + + /** + * Suspend the service running on given host + */ + public boolean suspend(ServiceType service, String hostname) throws IOException { + return signal(service, SIGSTOP, hostname); + } + + /** + * Resume the services running on given hosts + */ + public boolean resume(ServiceType service, String hostname) throws IOException { + return signal(service, SIGCONT, hostname); + } + + /** + * Returns whether the service is running on the remote host. This only checks whether the + * service still has a pid. + */ + public abstract boolean isRunning(ServiceType service, String hostname) throws IOException; + + /* TODO: further API ideas: + * + * //return services running on host: + * ServiceType[] getRunningServicesOnHost(String hostname); + * + * //return which services can be run on host (for example, to query whether hmaster can run on this host) + * ServiceType[] getRunnableServicesOnHost(String hostname); + * + * //return which hosts can run this service + * String[] getRunnableHostsForService(ServiceType service); + */ + +} \ No newline at end of file diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java new file mode 100644 index 0000000..3803bde --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.AdminProtocol; +import org.apache.hadoop.hbase.client.ClientProtocol; +import org.apache.hadoop.hbase.ipc.HMasterInterface; + +/** + * This class defines methods that can help with managing HBase clusters + * from unit tests and system tests. There are 3 types of cluster deployments: + * + *

+ * HBaseCluster unifies the way tests interact with the cluster, so that the same test can + * be run against a mini-cluster during unit test execution, or a real cluster having + * tens/hundreds of nodes during execution of integration tests. + * + *

+ * HBaseCluster exposes client-side public interfaces to tests, so that tests does not assume + * running in a particular mode. Not all the tests are suitable to be run on an actual cluster, + * and some tests will still need to mock stuff and introspect internal state. For those use + * cases from unit tests, or if more control is needed, you can use the subclasses directly. + * In that sense, this class does not abstract away every interface that MiniHBaseCluster + * or RealHBaseCluster provide. + */ +@InterfaceAudience.Private +public abstract class HBaseCluster implements Closeable, Configurable { + static final Log LOG = LogFactory.getLog(HBaseCluster.class.getName()); + protected Configuration conf; + + /** the status of the cluster before we begin */ + protected ClusterStatus initialClusterStatus; + + /** + * Construct an HBaseCluster + * @param conf Configuration to be used for cluster + */ + public HBaseCluster(Configuration conf) { + setConf(conf); + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + /** + * Returns a ClusterStatus for this HBase cluster. + * @see #getInitialClusterStatus() + */ + public abstract ClusterStatus getClusterStatus() throws IOException; + + /** + * Returns a ClusterStatus for this HBase cluster as observed at the + * starting of the HBaseCluster + */ + public ClusterStatus getInitialClusterStatus() throws IOException { + return initialClusterStatus; + } + + /** + * Returns an AdminProtocol interface to the regionserver + */ + public abstract AdminProtocol getAdminProtocol(ServerName serverName) throws IOException; + + /** + * Returns a ClientProtocol interface to the regionserver + */ + public abstract ClientProtocol getClientProtocol(ServerName serverName) throws IOException; + + /** + * Starts a new region server on the given hostname or if this is a mini/local cluster, + * starts a region server locally. + * @param hostname the hostname to start the regionserver on + * @return whether the operation finished with success + */ + public abstract boolean startRegionServer(String hostname) throws IOException; + + /** + * If this is a real cluster, kills the region server, otherwise, + * this causes the region server to exit doing basic clean up only. + * @return whether the operation finished with success + */ + public abstract boolean abortRegionServer(ServerName serverName) throws IOException; + + /** + * Stops the given region server, by attempting a gradual stop. + * @return whether the operation finished with success + */ + public abstract boolean stopRegionServer(ServerName serverName) throws IOException; + + /** + * Wait for the specified region server to join the cluster + * @return whether the operation finished with success + */ + public boolean waitForRegionServerToStart(String hostname, long timeout) + throws IOException { + long start = System.currentTimeMillis(); + while ((System.currentTimeMillis() - start) < timeout) { + for (ServerName server : getClusterStatus().getServers()) { + if (server.getHostname().equals(hostname)) { + return true; + } + } + } + return false; + } + + /** + * Wait for the specified region server to stop the thread / process. + * @return whether the operation finished with success + */ + public abstract boolean waitForRegionServerToStop(ServerName serverName, long timeout) + throws IOException; + + /** + * Returns an HMasterInterface to the active master + */ + public abstract HMasterInterface getMasterInterface() + throws ZooKeeperConnectionException, MasterNotRunningException; + + /** + * Starts a new master on the given hostname or if this is a mini/local cluster, + * starts a master locally. + * @param hostname the hostname to start the master on + * @return whether the operation finished with success + */ + public abstract boolean startMaster(String hostname) throws IOException; + + /** + * If this is a real cluster, kills the master, otherwise, + * this causes master to exit doing basic clean up only. + * @return whether the operation finished with success + */ + public abstract boolean abortMaster(ServerName serverName) throws IOException; + + /** + * Stops the given master, by attempting a gradual stop. + * @return whether the operation finished with success + */ + public abstract boolean stopMaster(ServerName serverName) throws IOException; + + /** + * Wait for the specified master to stop the thread / process. + * @return whether the operation finished with success + */ + public abstract boolean waitForMasterToStop(ServerName serverName, long timeout) throws IOException; + + /** + * Blocks until there is an active master and that master has completed + * initialization. + * + * @return true if an active master becomes available. false if there are no + * masters left. + * @throws InterruptedException + */ + public boolean waitForActiveAndReadyMaster() + throws IOException { + return waitForActiveAndReadyMaster(Long.MAX_VALUE); + } + + /** + * Blocks until there is an active master and that master has completed + * initialization. + * @param timeout the timeout limit in ms + * @return true if an active master becomes available. false if there are no + * masters left. + */ + public abstract boolean waitForActiveAndReadyMaster(long timeout) + throws IOException; + + /** + * Wait for HBase Cluster to shut down. + */ + public abstract void waitUntilShutDown(); + + /** + * Shut down the HBase cluster + */ + public abstract void shutdown() throws IOException; + + /** + * Restores the cluster to it's initial state if this is a real cluster, + * otherwise does nothing. + */ + public void restoreInitialStatus() throws IOException { + restoreClusterStatus(getInitialClusterStatus()); + } + + /** + * Restores the cluster to given state if this is a real cluster, + * otherwise does nothing. + */ + public void restoreClusterStatus(ClusterStatus desiredStatus) throws IOException { + } + + /** + * Get the ServerName of region server serving ROOT region + */ + public ServerName getServerHoldingRoot() throws IOException { + return getServerHoldingRegion(HRegionInfo.ROOT_REGIONINFO.getRegionName()); + } + + /** + * Get the ServerName of region server serving the first META region + */ + public ServerName getServerHoldingMeta() throws IOException { + return getServerHoldingRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + } + + /** + * Get the ServerName of region server serving the specified region + * @param regionName Name of the region in bytes + * @return ServerName that hosts the region or null + */ + public abstract ServerName getServerHoldingRegion(byte[] regionName) throws IOException; + + /** + * @return whether we are interacting with a real cluster as opposed to an in-process mini + * cluster/local cluster. + */ + public boolean isRealCluster() { + return false; + } +} \ No newline at end of file diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java new file mode 100644 index 0000000..9a8d921 --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HBaseClusterManager.CommandProvider.Operation; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.util.Shell; + +/** + * A default cluster manager for HBase. Uses SSH, and hbase shell scripts + * to manage the cluster. Assumes Unix-like commands are available like 'ps', + * 'kill', etc. Also assumes the user running the test has enough "power" to start & stop + * servers on the remote machines (for example, the test user could be the same user as the + * user the daemon isrunning as) + */ +@InterfaceAudience.Private +public class HBaseClusterManager extends ClusterManager { + + /** + * Executes commands over SSH + */ + static class RemoteShell extends Shell.ShellCommandExecutor { + + private String hostname; + + private String sshCmd = "/usr/bin/ssh"; + private String sshOptions = System.getenv("HBASE_SSH_OPTS"); //from conf/hbase-env.sh + + public RemoteShell(String hostname, String[] execString, File dir, Map env, + long timeout) { + super(execString, dir, env, timeout); + this.hostname = hostname; + } + + public RemoteShell(String hostname, String[] execString, File dir, Map env) { + super(execString, dir, env); + this.hostname = hostname; + } + + public RemoteShell(String hostname, String[] execString, File dir) { + super(execString, dir); + this.hostname = hostname; + } + + public RemoteShell(String hostname, String[] execString) { + super(execString); + this.hostname = hostname; + } + + @Override + protected String[] getExecString() { + return new String[] { + "bash", "-c", + StringUtils.join(new String[] { sshCmd, + sshOptions == null ? "" : sshOptions, + hostname, + "\"" + StringUtils.join(super.getExecString(), " ") + "\"" + }, " ")}; + } + + @Override + public void execute() throws IOException { + super.execute(); + } + + public void setSshCmd(String sshCmd) { + this.sshCmd = sshCmd; + } + + public void setSshOptions(String sshOptions) { + this.sshOptions = sshOptions; + } + + public String getSshCmd() { + return sshCmd; + } + + public String getSshOptions() { + return sshOptions; + } + } + + /** + * Provides command strings for services to be executed by Shell + */ + static abstract class CommandProvider { + + enum Operation { + START, STOP, RESTART + } + + public abstract String getCommand(ServiceType service, Operation op); + + public String isRunningCommand(ServiceType service) { + return findPidCommand(service); + } + + protected String findPidCommand(ServiceType service) { + return String.format("ps aux | grep %s | grep -v grep | tr -s ' ' | cut -d ' ' -f2", + service); + } + + public String signalCommand(ServiceType service, String signal) { + return String.format("%s | xargs kill -s %s", findPidCommand(service), signal); + } + } + + /** + * CommandProvider to manage the service as a System V service. + */ + static class SystemVServiceCommandProvider extends CommandProvider { + //TODO: this assumes root + @Override + public String getCommand(ServiceType service, Operation op) { + return String.format("service %s %s", service, op.toString().toLowerCase()); + } + } + + /** + * CommandProvider to manage the service using bin/hbase-* scripts + */ + static class HBaseShellCommandProvider extends CommandProvider { + private String getHBaseHome() { + return System.getenv("HBASE_HOME"); + } + + private String getConfig() { + String confDir = System.getenv("HBASE_CONF_DIR"); + if (confDir != null) { + return String.format("--config %s", confDir); + } + return ""; + } + + @Override + public String getCommand(ServiceType service, Operation op) { + return String.format("%s/bin/hbase-daemon.sh %s %s %s", getHBaseHome(), getConfig(), + op.toString().toLowerCase(), service); + } + } + + public HBaseClusterManager() { + super(); + } + + protected CommandProvider getCommandProvider(ServiceType service) { + //TODO: make it pluggable, or auto-detect the best command provider, should work with + //hadoop daemons as well + return new HBaseShellCommandProvider(); + } + + /** + * Execute the given command on the host using SSH + * @return pair of exit code and command output + */ + private Pair exec(String hostname, String... cmd) { + LOG.info("Executing remote command: " + StringUtils.join(cmd, " ") + " ,hostname:" + hostname); + RemoteShell shell = new RemoteShell(hostname, cmd); + try { + shell.execute(); + } catch(IOException ex) { + //we do not want to throw this + LOG.warn(ex.getMessage()); + } + LOG.info("Executed remote command, exit code:" + shell.getExitCode() + + " ,output:" + shell.getOutput()); + + return new Pair(shell.getExitCode(), shell.getOutput()); + } + + private boolean exec(String hostname, ServiceType service, Operation op) { + return exec(hostname, getCommandProvider(service).getCommand(service, op)).getFirst() == 0; + } + + @Override + public boolean start(ServiceType service, String hostname) throws IOException { + return exec(hostname, service, Operation.START); + } + + @Override + public boolean stop(ServiceType service, String hostname) throws IOException { + return exec(hostname, service, Operation.STOP); + } + + @Override + public boolean restart(ServiceType service, String hostname) throws IOException { + return exec(hostname, service, Operation.RESTART); + } + + @Override + public boolean signal(ServiceType service, String signal, String hostname) throws IOException { + return exec(hostname, getCommandProvider(service).signalCommand(service, signal)) + .getFirst() == 0; + } + + @Override + public boolean isRunning(ServiceType service, String hostname) throws IOException { + return exec(hostname, getCommandProvider(service).isRunningCommand(service)) + .getFirst() == 0; + } + +} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index be932d7..1ee0a36 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -102,7 +102,9 @@ import org.apache.zookeeper.ZooKeeper; * old HBaseTestCase and HBaseClusterTestCase functionality. * Create an instance and keep it around testing HBase. This class is * meant to be your one-stop shop for anything you might need testing. Manages - * one cluster at a time only. + * one cluster at a time only. Managed cluster can be an in-process + * {@link MiniHBaseCluster}, or a deployed cluster of type {@link RealHBaseCluster}. + * Not all methods work with the real cluster. * Depends on log4j being on classpath and * hbase-site.xml for logging and test-run configuration. It does not set * logging levels nor make changes to configuration parameters. @@ -127,7 +129,7 @@ public class HBaseTestingUtility { private boolean passedZkCluster = false; private MiniDFSCluster dfsCluster = null; - private MiniHBaseCluster hbaseCluster = null; + private HBaseCluster hbaseCluster = null; private MiniMRCluster mrCluster = null; /** If there is a mini cluster running for this testing utility instance. */ @@ -157,6 +159,11 @@ public class HBaseTestingUtility { "test.build.data.basedirectory"; /** + * Configuration that controls whether this utility assumes a running cluster + */ + public static final String IS_REAL_CLUSTER = "hbase.cluster.real"; + + /** * Default base directory for test output. */ public static final String DEFAULT_BASE_TEST_DIRECTORY = "target/test-data"; @@ -210,6 +217,11 @@ public class HBaseTestingUtility { public HBaseTestingUtility(Configuration conf) { this.conf = conf; + try { + this.createHBaseCluster(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } // a hbase checksum verification failure will cause unit tests to fail ChecksumUtil.generateExceptionForChecksumFailureForTest(true); @@ -230,6 +242,43 @@ public class HBaseTestingUtility { return this.conf; } + /** System and integration tests can be run against a "real" cluster as opposed to a mini cluster.*/ + private void createHBaseCluster() throws IOException { + boolean isRealCluster = false; + isRealCluster = Boolean.parseBoolean(System.getProperty(IS_REAL_CLUSTER, "false")); + if (!isRealCluster) { + isRealCluster = conf.getBoolean(IS_REAL_CLUSTER, false); + } + + if (!isRealCluster) { + return; + } + + + ClusterManager clusterManager = new HBaseClusterManager(); + this.hbaseCluster = new RealHBaseCluster(conf, clusterManager); + getHBaseAdmin(); + } + + /** + * Sets the configuration property to use a real cluster for the tests + */ + public static void setUseRealCluster(Configuration conf) { + conf.setBoolean(IS_REAL_CLUSTER, true); + System.setProperty(IS_REAL_CLUSTER, "true"); + } + + /** + * @return whether we are interacting with a real cluster as opposed to and in-process mini + * cluster or a local cluster. + */ + public boolean isRealCluster() { + if (hbaseCluster != null) { + return hbaseCluster.isRealCluster(); + } + return false; + } + /** * @return Where to write test data on local filesystem; usually * {@link #DEFAULT_BASE_TEST_DIRECTORY} @@ -671,7 +720,7 @@ public class HBaseTestingUtility { getHBaseAdmin(); // create immediately the hbaseAdmin LOG.info("Minicluster is up"); - return this.hbaseCluster; + return (MiniHBaseCluster)this.hbaseCluster; } /** @@ -699,7 +748,11 @@ public class HBaseTestingUtility { * @see #startMiniCluster() */ public MiniHBaseCluster getMiniHBaseCluster() { - return this.hbaseCluster; + if (this.hbaseCluster instanceof MiniHBaseCluster) { + return (MiniHBaseCluster)this.hbaseCluster; + } + throw new RuntimeException(hbaseCluster + " not an instance of " + + MiniHBaseCluster.class.getName()); } /** @@ -738,7 +791,7 @@ public class HBaseTestingUtility { if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); // Wait till hbase is down before going on to shutdown zk. - this.hbaseCluster.join(); + this.hbaseCluster.waitUntilShutDown(); this.hbaseCluster = null; } } @@ -776,7 +829,7 @@ public class HBaseTestingUtility { * @throws IOException */ public void flush() throws IOException { - this.hbaseCluster.flushcache(); + getMiniHBaseCluster().flushcache(); } /** @@ -784,7 +837,7 @@ public class HBaseTestingUtility { * @throws IOException */ public void flush(byte [] tableName) throws IOException { - this.hbaseCluster.flushcache(tableName); + getMiniHBaseCluster().flushcache(tableName); } @@ -1159,9 +1212,10 @@ public class HBaseTestingUtility { HConnection conn = table.getConnection(); conn.clearRegionCache(); // assign all the new regions IF table is enabled. - if (getHBaseAdmin().isTableEnabled(table.getTableName())) { + HBaseAdmin admin = getHBaseAdmin(); + if (admin.isTableEnabled(table.getTableName())) { for(HRegionInfo hri : newRegions) { - hbaseCluster.getMaster().assignRegion(hri); + admin.assign(hri.getRegionName()); } } @@ -1272,8 +1326,8 @@ public class HBaseTestingUtility { Bytes.toString(tableName)); byte [] firstrow = metaRows.get(0); LOG.debug("FirstRow=" + Bytes.toString(firstrow)); - int index = hbaseCluster.getServerWith(firstrow); - return hbaseCluster.getRegionServerThreads().get(index).getRegionServer(); + int index = getMiniHBaseCluster().getServerWith(firstrow); + return getMiniHBaseCluster().getRegionServerThreads().get(index).getRegionServer(); } /** @@ -1343,7 +1397,7 @@ public class HBaseTestingUtility { // Needed for TestImportTsv. conf.set("mapred.job.tracker", jobConf.get("mapred.job.tracker")); - // this for mrv2 support; mr1 ignores this + // this for mrv2 support; mr1 ignores this conf.set("mapreduce.framework.name", "yarn"); String rmAdress = jobConf.get("yarn.resourcemanager.address"); if (rmAdress != null) { @@ -1389,7 +1443,7 @@ public class HBaseTestingUtility { * @throws Exception */ public void expireMasterSession() throws Exception { - HMaster master = hbaseCluster.getMaster(); + HMaster master = getMiniHBaseCluster().getMaster(); expireSession(master.getZooKeeper(), false); } @@ -1399,7 +1453,7 @@ public class HBaseTestingUtility { * @throws Exception */ public void expireRegionServerSession(int index) throws Exception { - HRegionServer rs = hbaseCluster.getRegionServer(index); + HRegionServer rs = getMiniHBaseCluster().getRegionServer(index); expireSession(rs.getZooKeeper(), false); } @@ -1463,13 +1517,16 @@ public class HBaseTestingUtility { } } - /** - * Get the HBase cluster. + * Get the Mini HBase cluster. * * @return hbase cluster */ public MiniHBaseCluster getHBaseCluster() { + return getMiniHBaseCluster(); + } + + public HBaseCluster getHBaseClusterShim() { return hbaseCluster; } @@ -1647,8 +1704,8 @@ public class HBaseTestingUtility { public boolean ensureSomeRegionServersAvailable(final int num) throws IOException { boolean startedServer = false; - - for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i + *

  • Possibly takes hours to complete
  • + *
  • Can be run on a mini cluster or an actual cluster
  • + *
  • Can make changes to the given cluster (starting stopping daemons, etc)
  • + *
  • Should not be run in parallel of other integration tests
  • + * + * + * Integration / System tests should have a class name starting with "IntegrationTest" + * + * @see SmallTests + * @see MediumTests + * @see LargeTests + */ +public interface IntegrationTests { +} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/LargeTests.java hbase-server/src/test/java/org/apache/hadoop/hbase/LargeTests.java index f1b46fa..da9abda 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/LargeTests.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/LargeTests.java @@ -33,6 +33,7 @@ package org.apache.hadoop.hbase; * * @see SmallTests * @see MediumTests + * @see IntegrationTests */ public interface LargeTests { } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/MediumTests.java hbase-server/src/test/java/org/apache/hadoop/hbase/MediumTests.java index bbbde7c..24a44b9 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/MediumTests.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/MediumTests.java @@ -32,6 +32,7 @@ package org.apache.hadoop.hbase; * * @see SmallTests * @see LargeTests + * @see IntegrationTests */ public interface MediumTests { } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index c7442ae..5b01e24 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -30,18 +30,20 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.client.AdminProtocol; +import org.apache.hadoop.hbase.client.ClientProtocol; import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.ipc.HMasterInterface; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; -import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.io.MapWritable; /** * This class creates a single process HBase cluster. @@ -51,9 +53,8 @@ import org.apache.hadoop.io.MapWritable; */ @InterfaceAudience.Public @InterfaceStability.Evolving -public class MiniHBaseCluster { +public class MiniHBaseCluster extends HBaseCluster { static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName()); - private Configuration conf; public LocalHBaseCluster hbaseCluster; private static int index; @@ -78,9 +79,10 @@ public class MiniHBaseCluster { public MiniHBaseCluster(Configuration conf, int numMasters, int numRegionServers) throws IOException, InterruptedException { - this.conf = conf; + super(conf); conf.set(HConstants.MASTER_PORT, "0"); init(numMasters, numRegionServers); + this.initialClusterStatus = getClusterStatus(); } public Configuration getConfiguration() { @@ -187,7 +189,7 @@ public class MiniHBaseCluster { } private void init(final int nMasterNodes, final int nRegionNodes) - throws IOException, InterruptedException { + throws IOException { try { // start up a LocalHBaseCluster hbaseCluster = new LocalHBaseCluster(conf, nMasterNodes, 0, @@ -212,6 +214,56 @@ public class MiniHBaseCluster { } } + @Override + public boolean startRegionServer(String hostname) throws IOException { + RegionServerThread rs = this.startRegionServer(); + return rs != null; + } + + @Override + public boolean abortRegionServer(ServerName serverName) throws IOException { + abortRegionServer(getRegionServerIndex(serverName)); + return true; + } + + @Override + public boolean stopRegionServer(ServerName serverName) throws IOException { + stopRegionServer(getRegionServerIndex(serverName)); + return true; + } + + @Override + public boolean waitForRegionServerToStop(ServerName serverName, long timeout) throws IOException { + //ignore timeout for now + waitOnRegionServer(getRegionServerIndex(serverName)); + return true; + } + + @Override + public boolean startMaster(String hostname) throws IOException { + MasterThread master = this.startMaster(); + return master != null; + } + + @Override + public boolean abortMaster(ServerName serverName) throws IOException { + abortMaster(getMasterIndex(serverName)); + return true; + } + + @Override + public boolean stopMaster(ServerName serverName) throws IOException { + stopMaster(getMasterIndex(serverName)); + return true; + } + + @Override + public boolean waitForMasterToStop(ServerName serverName, long timeout) throws IOException { + //ignore timeout for now + waitOnMaster(getMasterIndex(serverName)); + return true; + } + /** * Starts a region server thread running * @@ -308,6 +360,13 @@ public class MiniHBaseCluster { } /** + * Return the master interface + */ + public HMasterInterface getMasterInterface() { + return this.hbaseCluster.getActiveMaster(); + } + + /** * Returns the current active master, if available. * @return the active HMaster, null if none is active. */ @@ -381,15 +440,21 @@ public class MiniHBaseCluster { * masters left. * @throws InterruptedException */ - public boolean waitForActiveAndReadyMaster() throws InterruptedException { + public boolean waitForActiveAndReadyMaster(long timeout) throws IOException { List mts; - while (!(mts = getMasterThreads()).isEmpty()) { + long start = System.currentTimeMillis(); + while (!(mts = getMasterThreads()).isEmpty() + && (System.currentTimeMillis() - start) < timeout) { for (JVMClusterUtil.MasterThread mt : mts) { if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) { return true; } } - Thread.sleep(100); + try { + Thread.sleep(100); + } catch (InterruptedException ex) { + //ignore + } } return false; } @@ -426,6 +491,15 @@ public class MiniHBaseCluster { HConnectionManager.deleteAllConnections(false); } + @Override + public void close() throws IOException { + } + + @Override + public ClusterStatus getClusterStatus() throws IOException { + return getMaster().getClusterStatus(); + } + /** * Call flushCache on all regions on all participating regionservers. * @throws IOException @@ -520,6 +594,15 @@ public class MiniHBaseCluster { return index; } + @Override + public ServerName getServerHoldingRegion(byte[] regionName) throws IOException { + int index = getServerWith(regionName); + if (index < 0) { + return null; + } + return getRegionServer(index).getServerName(); + } + /** * Counts the total numbers of regions being served by the currently online * region servers by asking each how many regions they have. Does not look @@ -546,4 +629,40 @@ public class MiniHBaseCluster { masterThread.getMaster().abort("killAll", new Throwable()); } } + + @Override + public void waitUntilShutDown() { + this.hbaseCluster.join(); + } + + protected int getRegionServerIndex(ServerName serverName) { + //we have a small number of region servers, this should be fine for now. + List servers = getRegionServerThreads(); + for (int i=0; i < servers.size(); i++) { + if (servers.get(i).getRegionServer().getServerName().equals(serverName)) { + return i; + } + } + return -1; + } + + protected int getMasterIndex(ServerName serverName) { + List masters = getMasterThreads(); + for (int i = 0; i < masters.size(); i++) { + if (masters.get(i).getMaster().getServerName().equals(serverName)) { + return i; + } + } + return -1; + } + + @Override + public AdminProtocol getAdminProtocol(ServerName serverName) throws IOException { + return getRegionServer(getRegionServerIndex(serverName)); + } + + @Override + public ClientProtocol getClientProtocol(ServerName serverName) throws IOException { + return getRegionServer(getRegionServerIndex(serverName)); + } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/RealHBaseCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/RealHBaseCluster.java new file mode 100644 index 0000000..39f352d --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/RealHBaseCluster.java @@ -0,0 +1,292 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import java.util.HashMap; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterManager.ServiceType; +import org.apache.hadoop.hbase.client.AdminProtocol; +import org.apache.hadoop.hbase.client.ClientProtocol; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.ipc.HMasterInterface; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; + +import com.google.common.collect.Sets; + +/** + * Manages the interactions with an already deployed cluster for integration + * and system tests. + */ +@InterfaceAudience.Private +public class RealHBaseCluster extends HBaseCluster { + + private HBaseAdmin admin; + + private ClusterManager clusterManager; + + public RealHBaseCluster(Configuration conf, ClusterManager clusterManager) throws IOException { + super(conf); + this.clusterManager = clusterManager; + this.admin = new HBaseAdmin(conf); + this.initialClusterStatus = getClusterStatus(); + } + + public void setClusterManager(ClusterManager clusterManager) { + this.clusterManager = clusterManager; + } + + public ClusterManager getClusterManager() { + return clusterManager; + } + + /** + * Returns a ClusterStatus for this HBase cluster + * @throws IOException + */ + @Override + public ClusterStatus getClusterStatus() throws IOException { + return admin.getClusterStatus(); + } + + @Override + public ClusterStatus getInitialClusterStatus() throws IOException { + return initialClusterStatus; + } + + @Override + public void close() throws IOException { + if (this.admin != null) { + admin.close(); + } + } + + @Override + public AdminProtocol getAdminProtocol(ServerName serverName) throws IOException { + return admin.getConnection().getAdmin(serverName.getHostname(), serverName.getPort()); + } + + @Override + public ClientProtocol getClientProtocol(ServerName serverName) throws IOException { + return admin.getConnection().getClient(serverName.getHostname(), serverName.getPort()); + } + + @Override + public boolean startRegionServer(String hostname) throws IOException { + LOG.info("Starting RS on: " + hostname); + return clusterManager.start(ServiceType.HBASE_REGIONSERVER, hostname); + } + + @Override + public boolean abortRegionServer(ServerName serverName) throws IOException { + LOG.info("Aborting RS: " + serverName.getServerName()); + return clusterManager.kill(ServiceType.HBASE_REGIONSERVER, serverName.getHostname()); + } + + @Override + public boolean stopRegionServer(ServerName serverName) throws IOException { + LOG.info("Stopping RS: " + serverName.getServerName()); + return clusterManager.stop(ServiceType.HBASE_REGIONSERVER, serverName.getHostname()); + } + + @Override + public boolean waitForRegionServerToStop(ServerName serverName, long timeout) throws IOException { + LOG.info("Waiting RS to stop: " + serverName.getServerName()); + return waitServiceToStop(ServiceType.HBASE_REGIONSERVER, serverName, timeout); + } + + private boolean waitServiceToStop(ServiceType service, ServerName serverName, long timeout) + throws IOException { + long start = System.currentTimeMillis(); + + while ((System.currentTimeMillis() - start) < timeout) { + if (!clusterManager.isRunning(service, serverName.getHostname())) { + return true; + } + sleep(1000); + } + return false; + } + + private void sleep(long milis) { + try { + Thread.sleep(milis); + } catch (InterruptedException ex) { + //ignore + } + } + + @Override + public HMasterInterface getMasterInterface() + throws ZooKeeperConnectionException, MasterNotRunningException { + HConnection conn = HConnectionManager.getConnection(conf); + return conn.getMaster(); + } + + @Override + public boolean startMaster(String hostname) throws IOException { + LOG.info("Starting Master on: " + hostname); + return clusterManager.start(ServiceType.HBASE_MASTER, hostname); + } + + @Override + public boolean abortMaster(ServerName serverName) throws IOException { + LOG.info("Aborting Master: " + serverName.getServerName()); + return clusterManager.kill(ServiceType.HBASE_MASTER, serverName.getHostname()); + } + + @Override + public boolean stopMaster(ServerName serverName) throws IOException { + LOG.info("Stopping Master: " + serverName.getServerName()); + return clusterManager.stop(ServiceType.HBASE_MASTER, serverName.getHostname()); + } + + @Override + public boolean waitForMasterToStop(ServerName serverName, long timeout) throws IOException { + LOG.info("Waiting Master to stop: " + serverName.getServerName()); + return waitServiceToStop(ServiceType.HBASE_MASTER, serverName, timeout); + } + + @Override + public boolean waitForActiveAndReadyMaster(long timeout) throws IOException { + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < timeout) { + try { + getMasterInterface(); + return true; + } catch (MasterNotRunningException m) { + LOG.warn("Master not started yet " + m); + } catch (ZooKeeperConnectionException e) { + LOG.warn("Failed to connect to ZK " + e); + } + sleep(1000); + } + return false; + } + + @Override + public ServerName getServerHoldingRegion(byte[] regionName) throws IOException { + HConnection connection = admin.getConnection(); + HRegionLocation regionLoc = connection.locateRegion(regionName); + if (regionLoc == null) { + return null; + } + + AdminProtocol client = connection.getAdmin(regionLoc.getHostname(), regionLoc.getPort()); + ServerInfo info = ProtobufUtil.getServerInfo(client); + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn = info.getServerName(); + + return new ServerName(sn.getHostName(), sn.getPort(), sn.getStartCode()); + } + + @Override + public void waitUntilShutDown() { + //Simply wait for a few seconds for now (after issuing serverManager.kill + throw new RuntimeException("Not implemented yet"); + } + + @Override + public void shutdown() throws IOException { + //not sure we want this + throw new RuntimeException("Not implemented yet"); + } + + @Override + public boolean isRealCluster() { + return true; + } + + @Override + public void restoreClusterStatus(ClusterStatus initial) throws IOException { + //TODO: caution: not tested + ClusterStatus current = getClusterStatus(); + + //restore masters + + //check whether current master has changed + if (!ServerName.isSameHostnameAndPort(initial.getMaster(), current.getMaster())) { + //master has changed, we would like to undo this. + //1. Kill the current backups + //2. Stop current master + //3. Start a master at the initial hostname (if not already running as backup) + //4. Start backup masters + boolean foundOldMaster = false; + for (ServerName currentBackup : current.getBackupMasters()) { + if (!ServerName.isSameHostnameAndPort(currentBackup, initial.getMaster())) { + stopMaster(currentBackup); + } else { + foundOldMaster = true; + } + } + stopMaster(current.getMaster()); + if (foundOldMaster) { //if initial master is not running as a backup + startMaster(initial.getMaster().getHostname()); + } + waitForActiveAndReadyMaster(); //wait so that active master takes over + + //start backup masters + for (ServerName backup : initial.getBackupMasters()) { + //these are not started in backup mode, but we should already have an active master + startMaster(backup.getHostname()); + } + } else { + //current master has not changed, match up backup masters + HashMap initialBackups = new HashMap(); + HashMap currentBackups = new HashMap(); + + for (ServerName server : initial.getBackupMasters()) { + initialBackups.put(server.getHostname(), server); + } + for (ServerName server : current.getBackupMasters()) { + currentBackups.put(server.getHostname(), server); + } + + for (String hostname : Sets.difference(initialBackups.keySet(), currentBackups.keySet())) { + startMaster(hostname); + } + + for (String hostname : Sets.difference(currentBackups.keySet(), initialBackups.keySet())) { + stopMaster(currentBackups.get(hostname)); + } + } + + //restore region servers + HashMap initialServers = new HashMap(); + HashMap currentServers = new HashMap(); + + for (ServerName server : initial.getServers()) { + initialServers.put(server.getHostname(), server); + } + for (ServerName server : current.getServers()) { + currentServers.put(server.getHostname(), server); + } + + for (String hostname : Sets.difference(initialServers.keySet(), currentServers.keySet())) { + startRegionServer(hostname); + } + + for (String hostname : Sets.difference(currentServers.keySet(), initialServers.keySet())) { + stopRegionServer(currentServers.get(hostname)); + } + } +} \ No newline at end of file diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/SmallTests.java hbase-server/src/test/java/org/apache/hadoop/hbase/SmallTests.java index c702f5a..eba6a94 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/SmallTests.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/SmallTests.java @@ -29,6 +29,7 @@ package org.apache.hadoop.hbase; * * @see MediumTests * @see LargeTests + * @see IntegrationTests */ public interface SmallTests { } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java index d917bdb..4d797a3 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java @@ -18,9 +18,8 @@ package org.apache.hadoop.hbase; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runners.Suite; +import static junit.framework.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -31,8 +30,9 @@ import java.util.ArrayList; import java.util.Enumeration; import java.util.List; -import static junit.framework.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runners.Suite; /** @@ -42,7 +42,7 @@ import static org.junit.Assert.assertTrue; public class TestCheckTestClasses { /** - * Throws an assertion if we find a test class without category (small/medium/large). + * Throws an assertion if we find a test class without category (small/medium/large/integration). * List all the test classes without category in the assertion message. */ @Test @@ -50,7 +50,7 @@ public class TestCheckTestClasses { List> badClasses = new java.util.ArrayList>(); for (Class c : findTestClasses()) { - if (!existCategoryAnnotation(c)) { + if (!existCategoryAnnotation(c, null)) { badClasses.add(c); } } @@ -59,9 +59,22 @@ public class TestCheckTestClasses { + badClasses, badClasses.isEmpty()); } + /** Returns whether the class has @Category annotation having the xface value. + */ + private boolean existCategoryAnnotation(Class c, Class xface) { + Category category = c.getAnnotation(Category.class); - private boolean existCategoryAnnotation(Class c) { - return (c.getAnnotation(Category.class) != null); + if (category != null) { + if (xface == null) { + return true; + } + for (Class cc : category.value()) { + if (cc.equals(xface)) { + return true; + } + } + } + return false; } /* @@ -88,6 +101,19 @@ public class TestCheckTestClasses { return false; } + /** + * Finds test classes which are annotated with @Category having xface value + * @param xface the @Category value + */ + public List> findTestClasses(Class xface) throws ClassNotFoundException, IOException { + List> classes = new ArrayList>(); + for (Class c : findTestClasses()) { + if (existCategoryAnnotation(c, xface)) { + classes.add(c); + } + } + return classes; + } private List> findTestClasses() throws ClassNotFoundException, IOException { final String packageName = "org.apache.hadoop.hbase"; @@ -124,7 +150,8 @@ public class TestCheckTestClasses { final String fileName = file.getName(); if (file.isDirectory()) { classes.addAll(findTestClasses(file, packageName + "." + fileName)); - } else if (fileName.endsWith(".class") && fileName.startsWith("Test")) { + } else if (fileName.endsWith(".class") && + (fileName.startsWith("Test") || fileName.startsWith("IntegrationTest"))) { Class c = Class.forName( packageName + '.' + fileName.substring(0, fileName.length() - 6), false, diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index db47ccf..6edc086 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; +import org.apache.hadoop.hbase.util.SimpleStoppable; import org.apache.hadoop.hbase.util.Threads; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -202,20 +203,6 @@ public class TestEndToEndSplitTransaction { regionChecker.verify(); } - private static class SimpleStoppable implements Stoppable { - volatile boolean stopped = false; - - @Override - public void stop(String why) { - this.stopped = true; - } - - @Override - public boolean isStopped() { - return stopped; - } - } - static class RegionSplitter extends Thread { Throwable ex; HTable table; diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java index 9fa7acd..4b81b3b 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java @@ -289,7 +289,7 @@ public class LoadTestTool extends AbstractHBaseTool { } @Override - protected void doWork() throws IOException { + protected int doWork() throws IOException { if (cmd.hasOption(OPT_ZK_QUORUM)) { conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM)); } @@ -335,6 +335,16 @@ public class LoadTestTool extends AbstractHBaseTool { if (isRead) { readerThreads.waitForFinish(); } + + boolean success = true; + if (isWrite) { + success = success && writerThreads.getNumWriteFailures() == 0; + } + if (isRead) { + success = success && readerThreads.getNumReadErrors() == 0 + && readerThreads.getNumReadFailures() == 0; + } + return success ? 0 : 1; } public static void main(String[] args) { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java index 3128dc6..a2f3d28 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java @@ -86,7 +86,7 @@ public class RestartMetaTest extends AbstractHBaseTool { } @Override - protected void doWork() throws Exception { + protected int doWork() throws Exception { ProcessBasedLocalHBaseCluster hbaseCluster = new ProcessBasedLocalHBaseCluster(conf, NUM_DATANODES, numRegionServers); hbaseCluster.startMiniDFS(); @@ -128,6 +128,7 @@ public class RestartMetaTest extends AbstractHBaseTool { + Bytes.toStringBinary(result.getFamilyMap(HConstants.CATALOG_FAMILY) .get(HConstants.SERVER_QUALIFIER))); } + return 0; } @Override diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/SimpleStoppable.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/SimpleStoppable.java new file mode 100644 index 0000000..15d9472 --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/SimpleStoppable.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Stoppable; + +/** + * A simple implementation for a Stoppable service + */ +@InterfaceAudience.Private +public class SimpleStoppable implements Stoppable { + volatile boolean stopped = false; + + @Override + public void stop(String why) { + this.stopped = true; + } + + @Override + public boolean isStopped() { + return stopped; + } +} \ No newline at end of file