From a8c88bbc9012a8fccfb578fcc389b21fbf6017d8 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 20 Aug 2018 21:42:34 -0700 Subject: [PATCH] HBASE-21071 HBaseTestingUtility::startMiniCluster() to use builder pattern (part1) --- .../hadoop/hbase/HBaseTestingUtility.java | 485 +++++++++++------- .../hadoop/hbase/StartMiniClusterOption.java | 252 +++++++++ 2 files changed, 556 insertions(+), 181 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/StartMiniClusterOption.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index b938d2834f..a64f07254d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -765,197 +765,265 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { } } + /** + * Start up a minicluster of hbase, dfs, and zookeeper where WAL's walDir is created separately. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param createWALDir Whether to create a new WAL directory. + * @return The mini HBase cluster created. + * @see #shutdownMiniCluster() + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. + */ + @Deprecated + public MiniHBaseCluster startMiniCluster(boolean createWALDir) throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .createWALDir(createWALDir).build(); + return startMiniCluster(option); + } /** * Start up a minicluster of hbase, dfs, and zookeeper. - * @throws Exception - * @return Mini hbase cluster instance created. - * @see #shutdownMiniDFSCluster() + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numSlaves Slave node number, for both HBase region server and HDFS data node. + * @param createRootDir Whether to create a new root or data directory path. + * @return The mini HBase cluster created. + * @see #shutdownMiniCluster() + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster() throws Exception { - return startMiniCluster(1, 1); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numSlaves, boolean createRootDir) + throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numRegionServers(numSlaves).createRootDir(createRootDir).numDataNodes(numSlaves).build(); + return startMiniCluster(option); } /** - * Start up a minicluster of hbase, dfs, and zookeeper where WAL's walDir is created separately. - * @throws Exception - * @return Mini hbase cluster instance created. - * @see #shutdownMiniDFSCluster() + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numSlaves Slave node number, for both HBase region server and HDFS data node. + * @param createRootDir Whether to create a new root or data directory path. + * @param createWALDir Whether to create a new WAL directory. + * @return The mini HBase cluster created. + * @see #shutdownMiniCluster() + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster(boolean withWALDir) throws Exception { - return startMiniCluster(1, 1, 1, null, null, null, false, withWALDir); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numSlaves, boolean createRootDir, + boolean createWALDir) throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numRegionServers(numSlaves).numDataNodes(numSlaves).createRootDir(createRootDir) + .createWALDir(createWALDir).build(); + return startMiniCluster(option); } /** * Start up a minicluster of hbase, dfs, and zookeeper. - * Set the create flag to create root or data directory path or not - * (will overwrite if dir already exists) - * @throws Exception - * @return Mini hbase cluster instance created. - * @see #shutdownMiniDFSCluster() + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numSlaves Slave node number, for both HBase region server and HDFS data node. + * @param createRootDir Whether to create a new root or data directory path. + * @return The mini HBase cluster created. + * @see #shutdownMiniCluster() + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create) - throws Exception { - return startMiniCluster(1, numSlaves, create); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numSlaves, boolean createRootDir) + throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).numRegionServers(numSlaves).createRootDir(createRootDir) + .numDataNodes(numSlaves).build(); + return startMiniCluster(option); } /** - * Start up a minicluster of hbase, optionally dfs, and zookeeper. - * Modifies Configuration. Homes the cluster data directory under a random - * subdirectory in a directory under System property test.build.data. - * Directory is cleaned up on exit. - * @param numSlaves Number of slaves to start up. We'll start this many - * datanodes and regionservers. If numSlaves is > 1, then make sure - * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise - * bind errors. - * @throws Exception + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numSlaves Slave node number, for both HBase region server and HDFS data node. + * @return The mini HBase cluster created. * @see #shutdownMiniCluster() - * @return Mini hbase cluster instance created. + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster(final int numSlaves) - throws Exception { - return startMiniCluster(1, numSlaves, false); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numSlaves) throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).numRegionServers(numSlaves).numDataNodes(numSlaves).build(); + return startMiniCluster(option); } - public MiniHBaseCluster startMiniCluster(final int numSlaves, boolean create, boolean withWALDir) - throws Exception { - return startMiniCluster(1, numSlaves, numSlaves, null, null, null, create, withWALDir); + /** + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numSlaves Slave node number, for both HBase region server and HDFS data node. + * @param dataNodeHosts The hostnames of DataNodes to run on. If not null, its size will overwrite + * HDFS data node number. + * @param createRootDir Whether to create a new root or data directory path. + * @return The mini HBase cluster created. + * @see #shutdownMiniCluster() + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. + */ + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numSlaves, String[] dataNodeHosts, + boolean createRootDir) throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).numRegionServers(numSlaves).createRootDir(createRootDir) + .numDataNodes(numSlaves).dataNodeHosts(dataNodeHosts).build(); + return startMiniCluster(option); } /** - * Start minicluster. Whether to create a new root or data dir path even if such a path - * has been created earlier is decided based on flag create - * @throws Exception + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numSlaves Slave node number, for both HBase region server and HDFS data node. + * @param dataNodeHosts The hostnames of DataNodes to run on. If not null, its size will overwrite + * HDFS data node number. + * @return The mini HBase cluster created. * @see #shutdownMiniCluster() - * @return Mini hbase cluster instance created. + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves, boolean create) - throws Exception { - return startMiniCluster(numMasters, numSlaves, null, create); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numSlaves, String[] dataNodeHosts) + throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).numRegionServers(numSlaves) + .dataNodeHosts(dataNodeHosts).build(); + return startMiniCluster(option); } /** - * start minicluster - * @throws Exception + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numRegionServers Number of region servers. + * @param numDataNodes Number of datanodes. + * @return The mini HBase cluster created. * @see #shutdownMiniCluster() - * @return Mini hbase cluster instance created. + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves) - throws Exception { - return startMiniCluster(numMasters, numSlaves, null, false); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numRegionServers, int numDataNodes) + throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).numRegionServers(numRegionServers).numDataNodes(numDataNodes) + .build(); + return startMiniCluster(option); } - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves, final String[] dataNodeHosts, boolean create) - throws Exception { - return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, - null, null, create, false); - } - - /** - * Start up a minicluster of hbase, optionally dfs, and zookeeper. - * Modifies Configuration. Homes the cluster data directory under a random - * subdirectory in a directory under System property test.build.data. - * Directory is cleaned up on exit. - * @param numMasters Number of masters to start up. We'll start this many - * hbase masters. If numMasters > 1, you can find the active/primary master - * with {@link MiniHBaseCluster#getMaster()}. - * @param numSlaves Number of slaves to start up. We'll start this many - * regionservers. If dataNodeHosts == null, this also indicates the number of - * datanodes to start. If dataNodeHosts != null, the number of datanodes is - * based on dataNodeHosts.length. - * If numSlaves is > 1, then make sure - * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise - * bind errors. - * @param dataNodeHosts hostnames DNs to run on. - * This is useful if you want to run datanode on distinct hosts for things - * like HDFS block location verification. - * If you start MiniDFSCluster without host names, - * all instances of the datanodes will have the same host name. - * @throws Exception + /** + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numSlaves Slave node number, for both HBase region server and HDFS data node. + * @param dataNodeHosts The hostnames of DataNodes to run on. If not null, its size will overwrite + * HDFS data node number. + * @param masterClass The class to use as HMaster, or null for default. + * @param rsClass The class to use as HRegionServer, or null for default. + * @return The mini HBase cluster created. * @see #shutdownMiniCluster() - * @return Mini hbase cluster instance created. + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves, final String[] dataNodeHosts) throws Exception { - return startMiniCluster(numMasters, numSlaves, numSlaves, dataNodeHosts, - null, null); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numSlaves, String[] dataNodeHosts, + Class masterClass, + Class rsClass) + throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).masterClass(masterClass) + .numRegionServers(numSlaves).rsClass(rsClass) + .numDataNodes(numSlaves).dataNodeHosts(dataNodeHosts) + .build(); + return startMiniCluster(option); } /** - * Same as {@link #startMiniCluster(int, int)}, but with custom number of datanodes. - * @param numDataNodes Number of data nodes. + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numRegionServers Number of region servers. + * @param numDataNodes Number of datanodes. + * @param dataNodeHosts The hostnames of DataNodes to run on. If not null, its size will overwrite + * HDFS data node number. + * @param masterClass The class to use as HMaster, or null for default. + * @param rsClass The class to use as HRegionServer, or null for default. + * @return The mini HBase cluster created. + * @see #shutdownMiniCluster() + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. */ - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves, final int numDataNodes) throws Exception { - return startMiniCluster(numMasters, numSlaves, numDataNodes, null, null, null); + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numRegionServers, int numDataNodes, + String[] dataNodeHosts, Class masterClass, + Class rsClass) + throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).masterClass(masterClass) + .numRegionServers(numRegionServers).rsClass(rsClass) + .numDataNodes(numDataNodes).dataNodeHosts(dataNodeHosts) + .build(); + return startMiniCluster(option); } /** - * Start up a minicluster of hbase, optionally dfs, and zookeeper. - * Modifies Configuration. Homes the cluster data directory under a random - * subdirectory in a directory under System property test.build.data. - * Directory is cleaned up on exit. - * @param numMasters Number of masters to start up. We'll start this many - * hbase masters. If numMasters > 1, you can find the active/primary master - * with {@link MiniHBaseCluster#getMaster()}. - * @param numSlaves Number of slaves to start up. We'll start this many - * regionservers. If dataNodeHosts == null, this also indicates the number of - * datanodes to start. If dataNodeHosts != null, the number of datanodes is - * based on dataNodeHosts.length. - * If numSlaves is > 1, then make sure - * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise - * bind errors. - * @param dataNodeHosts hostnames DNs to run on. - * This is useful if you want to run datanode on distinct hosts for things - * like HDFS block location verification. - * If you start MiniDFSCluster without host names, - * all instances of the datanodes will have the same host name. - * @param masterClass The class to use as HMaster, or null for default - * @param regionserverClass The class to use as HRegionServer, or null for - * default - * @throws Exception + * Start up a minicluster of hbase, dfs, and zookeeper. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numRegionServers Number of region servers. + * @param numDataNodes Number of datanodes. + * @param dataNodeHosts The hostnames of DataNodes to run on. If not null, its size will overwrite + * HDFS data node number. + * @param masterClass The class to use as HMaster, or null for default. + * @param rsClass The class to use as HRegionServer, or null for default. + * @param createRootDir Whether to create a new root or data directory path. + * @param createWALDir Whether to create a new WAL directory. * @return The mini HBase cluster created. * @see #shutdownMiniCluster() - * @return Mini hbase cluster instance created. + * @deprecated Use {@link #startMiniCluster(StartMiniClusterOption)} instead. + */ + @Deprecated + public MiniHBaseCluster startMiniCluster(int numMasters, int numRegionServers, int numDataNodes, + String[] dataNodeHosts, Class masterClass, + Class rsClass, boolean createRootDir, + boolean createWALDir) throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).masterClass(masterClass) + .numRegionServers(numRegionServers).rsClass(rsClass) + .numDataNodes(numDataNodes).dataNodeHosts(dataNodeHosts) + .createRootDir(createRootDir).createWALDir(createWALDir) + .build(); + return startMiniCluster(option); + } + + /** + * Start up a minicluster of hbase, dfs and zookeeper clusters with given slave node number. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numSlaves slave node number, for both HBase region server and HDFS data node. + * @see #startMiniCluster(StartMiniClusterOption option) + * @see #shutdownMiniDFSCluster() */ - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves, final String[] dataNodeHosts, Class masterClass, - Class regionserverClass) - throws Exception { - return startMiniCluster( - numMasters, numSlaves, numSlaves, dataNodeHosts, masterClass, regionserverClass); + public MiniHBaseCluster startMiniCluster(int numSlaves) throws Exception { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numRegionServers(numSlaves).numDataNodes(numSlaves).build(); + return startMiniCluster(option); } - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves, int numDataNodes, final String[] dataNodeHosts, - Class masterClass, - Class regionserverClass) - throws Exception { - return startMiniCluster(numMasters, numSlaves, numDataNodes, dataNodeHosts, - masterClass, regionserverClass, false, false); + /** + * Start up a minicluster of hbase, dfs and zookeeper all using default options. + * Option default value can be found in {@link StartMiniClusterOption.Builder}. + * @see #startMiniCluster(StartMiniClusterOption option) + * @see #shutdownMiniDFSCluster() + */ + public MiniHBaseCluster startMiniCluster() throws Exception { + return startMiniCluster(StartMiniClusterOption.builder().build()); } /** - * Same as {@link #startMiniCluster(int, int, String[], Class, Class)}, but with custom - * number of datanodes. - * @param numDataNodes Number of data nodes. - * @param create Set this flag to create a new - * root or data directory path or not (will overwrite if exists already). + * Start up a mini cluster of hbase, optionally dfs and zookeeper if needed. + * It modifies Configuration. It homes the cluster data directory under a random + * subdirectory in a directory under System property test.build.data, to be cleaned up on exit. + * @see #shutdownMiniDFSCluster() */ - public MiniHBaseCluster startMiniCluster(final int numMasters, - final int numSlaves, int numDataNodes, final String[] dataNodeHosts, - Class masterClass, - Class regionserverClass, - boolean create, boolean withWALDir) - throws Exception { - if (dataNodeHosts != null && dataNodeHosts.length != 0) { - numDataNodes = dataNodeHosts.length; - } - - LOG.info("Starting up minicluster with " + numMasters + " master(s) and " + - numSlaves + " regionserver(s) and " + numDataNodes + " datanode(s)"); + public MiniHBaseCluster startMiniCluster(StartMiniClusterOption option) throws Exception { + LOG.info("Starting up minicluster with option: {}", option); // If we already put up a cluster, fail. if (miniClusterRunning) { @@ -968,54 +1036,35 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { // Bring up mini dfs cluster. This spews a bunch of warnings about missing // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'. - if(this.dfsCluster == null) { + if (dfsCluster == null) { LOG.info("STARTING DFS"); - dfsCluster = startMiniDFSCluster(numDataNodes, dataNodeHosts); - } else LOG.info("NOT STARTING DFS"); + dfsCluster = startMiniDFSCluster(option.getNumDataNodes(), option.getDataNodeHosts()); + } else { + LOG.info("NOT STARTING DFS"); + } // Start up a zk cluster. if (getZkCluster() == null) { - startMiniZKCluster(); + startMiniZKCluster(option.getNumZkServers()); } // Start the MiniHBaseCluster - return startMiniHBaseCluster(numMasters, numSlaves, null, masterClass, - regionserverClass, create, withWALDir); - } - - public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves) - throws IOException, InterruptedException { - return startMiniHBaseCluster(numMasters, numSlaves, null); - } - - public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, final int numSlaves, - List rsPorts) throws IOException, InterruptedException { - return startMiniHBaseCluster(numMasters, numSlaves, rsPorts, null, null, false, false); + return startMiniHBaseCluster(option); } /** - * Starts up mini hbase cluster. Usually used after call to - * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters. + * Starts up mini hbase cluster. * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. - * @param rsPorts Ports that RegionServer should use; pass ports if you want to test cluster - * restart where for sure the regionservers come up on same address+port (but - * just with different startcode); by default mini hbase clusters choose new - * arbitrary ports on each cluster start. - * @param create Whether to create a - * root or data directory path or not; will overwrite if exists already. + * This is useful when doing stepped startup of clusters. * @return Reference to the hbase mini hbase cluster. - * @throws IOException - * @throws InterruptedException - * @see #startMiniCluster() + * @see #startMiniCluster(StartMiniClusterOption) + * @see #shutdownMiniHBaseCluster() */ - public MiniHBaseCluster startMiniHBaseCluster(final int numMasters, - final int numSlaves, List rsPorts, Class masterClass, - Class regionserverClass, - boolean create, boolean withWALDir) - throws IOException, InterruptedException { + public MiniHBaseCluster startMiniHBaseCluster(StartMiniClusterOption option) + throws IOException, InterruptedException { // Now do the mini hbase cluster. Set the hbase.rootdir in config. - createRootDir(create); - if (withWALDir) { + createRootDir(option.isCreateRootDir()); + if (option.isCreateWALDir()) { createWALRootDir(); } // Set the hbase.fs.tmp.dir config to make sure that we have some default value. This is @@ -1025,16 +1074,17 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { // These settings will make the server waits until this exact number of // regions servers are connected. if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1) == -1) { - conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, numSlaves); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, option.getNumRegionServers()); } if (conf.getInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1) == -1) { - conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, numSlaves); + conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, option.getNumRegionServers()); } Configuration c = new Configuration(this.conf); TraceUtil.initTracer(c); this.hbaseCluster = - new MiniHBaseCluster(c, numMasters, numSlaves, rsPorts, masterClass, regionserverClass); + new MiniHBaseCluster(c, option.getNumMasters(), option.getNumRegionServers(), + option.getRsPorts(), option.getMasterClass(), option.getRsClass()); // Don't leave here till we've done a successful scan of the hbase:meta Table t = getConnection().getTable(TableName.META_TABLE_NAME); ResultScanner s = t.getScanner(new Scan()); @@ -1045,9 +1095,82 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { t.close(); getAdmin(); // create immediately the hbaseAdmin - LOG.info("Minicluster is up; activeMaster=" + this.getHBaseCluster().getMaster()); + LOG.info("Minicluster is up; activeMaster={}", getHBaseCluster().getMaster()); + + return (MiniHBaseCluster) hbaseCluster; + } + + /** + * Starts up mini hbase cluster using default options. + * Default options can be found in {@link StartMiniClusterOption.Builder}. + * @see #startMiniHBaseCluster(StartMiniClusterOption) + * @see #shutdownMiniHBaseCluster() + */ + public MiniHBaseCluster startMiniHBaseCluster() throws IOException, InterruptedException { + return startMiniHBaseCluster(StartMiniClusterOption.builder().build()); + } - return (MiniHBaseCluster)this.hbaseCluster; + /** + * Starts up mini hbase cluster. + * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numRegionServers Number of region servers. + * @return The mini HBase cluster created. + * @see #shutdownMiniHBaseCluster() + * @deprecated Use {@link #startMiniHBaseCluster(StartMiniClusterOption)} instead. + */ + @Deprecated + public MiniHBaseCluster startMiniHBaseCluster(int numMasters, int numRegionServers) + throws IOException, InterruptedException { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).numRegionServers(numRegionServers).build(); + return startMiniHBaseCluster(option); + } + + /** + * Starts up mini hbase cluster. + * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numRegionServers Number of region servers. + * @param rsPorts Ports that RegionServer should use. + * @return The mini HBase cluster created. + * @see #shutdownMiniHBaseCluster() + * @deprecated Use {@link #startMiniHBaseCluster(StartMiniClusterOption)} instead. + */ + @Deprecated + public MiniHBaseCluster startMiniHBaseCluster(int numMasters, int numRegionServers, + List rsPorts) throws IOException, InterruptedException { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).numRegionServers(numRegionServers).rsPorts(rsPorts).build(); + return startMiniHBaseCluster(option); + } + + /** + * Starts up mini hbase cluster. + * Usually you won't want this. You'll usually want {@link #startMiniCluster()}. + * All other options will use default values, defined in {@link StartMiniClusterOption.Builder}. + * @param numMasters Master node number. + * @param numRegionServers Number of region servers. + * @param rsPorts Ports that RegionServer should use. + * @param masterClass The class to use as HMaster, or null for default. + * @param rsClass The class to use as HRegionServer, or null for default. + * @param createRootDir Whether to create a new root or data directory path. + * @param createWALDir Whether to create a new WAL directory. * @return The mini HBase cluster created. + * @see #shutdownMiniHBaseCluster() + * @deprecated Use {@link #startMiniHBaseCluster(StartMiniClusterOption)} instead. + */ + @Deprecated + public MiniHBaseCluster startMiniHBaseCluster(int numMasters, int numRegionServers, + List rsPorts, Class masterClass, + Class rsClass, + boolean createRootDir, boolean createWALDir) throws IOException, InterruptedException { + StartMiniClusterOption option = StartMiniClusterOption.builder() + .numMasters(numMasters).masterClass(masterClass) + .numRegionServers(numRegionServers).rsClass(rsClass).rsPorts(rsPorts) + .createRootDir(createRootDir).createWALDir(createWALDir).build(); + return startMiniHBaseCluster(option); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/StartMiniClusterOption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartMiniClusterOption.java new file mode 100644 index 0000000000..dd93563b2b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartMiniClusterOption.java @@ -0,0 +1,252 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.master.HMaster; + +/** + * Options for starting up a mini cluster (including an hbase, dfs and zookeeper clusters) in test. + * The options include HDFS options to build mini dfs cluster, Zookeeper options to build mini zk + * cluster, and mostly HBase options to build mini hbase cluster. + * + * To create an object, use a {@link Builder}. + * Example usage: + *
+ *    StartMiniClusterOption option = StartMiniClusterOption.builder().
+ *        .numMasters(3).rsClass(MyRegionServer.class).createWALDir(true).build();
+ * 
+ * + * Default values can be found in {@link Builder}. + */ +public final class StartMiniClusterOption { + /** + * Number of masters to start up. We'll start this many hbase masters. If numMasters > 1, you + * can find the active/primary master with {@link MiniHBaseCluster#getMaster()}. + */ + private final int numMasters; + /** + * The class to use as HMaster, or null for default. + */ + private final Class masterClass; + + /** + * Number of region servers to start up. + * If this value is > 1, then make sure config "hbase.regionserver.info.port" is -1 + * (i.e. no ui per regionserver) otherwise bind errors. + */ + private final int numRegionServers; + /** + * Ports that RegionServer should use. Pass ports if you want to test cluster restart where for + * sure the regionservers come up on same address+port (but just with different startcode); by + * default mini hbase clusters choose new arbitrary ports on each cluster start. + */ + private final List rsPorts; + /** + * The class to use as HRegionServer, or null for default. + */ + private Class rsClass; + + /** + * Number of datanodes. Used to create mini DSF cluster. Surpassed by {@link #dataNodeHosts} size. + */ + private final int numDataNodes; + /** + * The hostnames of DataNodes to run on. This is useful if you want to run datanode on distinct + * hosts for things like HDFS block location verification. If you start MiniDFSCluster without + * host names, all instances of the datanodes will have the same host name. + */ + private final String[] dataNodeHosts; + + /** + * Number of Zookeeper servers. + */ + private final int numZkServers; + + /** + * Whether to create a new root or data directory path. If true, the newly created data directory + * will be configured as HBase rootdir. This will overwrite existing root directory config. + */ + private final boolean createRootDir; + + /** + * Whether to create a new WAL directory. If true, the newly created directory will be configured + * as HBase wal.dir which is separate from HBase rootdir. + */ + private final boolean createWALDir; + + /** + * Private constructor. Use {@link Builder#build()}. + */ + private StartMiniClusterOption(int numMasters, Class masterClass, + int numRegionServers, List rsPorts, + Class rsClass, int numDataNodes, + String[] dataNodeHosts, int numZkServers, boolean createRootDir, boolean createWALDir) { + this.numMasters = numMasters; + this.masterClass = masterClass; + this.numRegionServers = numRegionServers; + this.rsPorts = rsPorts; + this.rsClass = rsClass; + this.numDataNodes = numDataNodes; + this.dataNodeHosts = dataNodeHosts; + this.numZkServers = numZkServers; + this.createRootDir = createRootDir; + this.createWALDir = createWALDir; + } + + public int getNumMasters() { + return numMasters; + } + + public Class getMasterClass() { + return masterClass; + } + + public int getNumRegionServers() { + return numRegionServers; + } + + public List getRsPorts() { + return rsPorts; + } + + public Class getRsClass() { + return rsClass; + } + + public int getNumDataNodes() { + return numDataNodes; + } + + public String[] getDataNodeHosts() { + return dataNodeHosts; + } + + public int getNumZkServers() { + return numZkServers; + } + + public boolean isCreateRootDir() { + return createRootDir; + } + + public boolean isCreateWALDir() { + return createWALDir; + } + + @Override + public String toString() { + return "StartMiniClusterOption{" + "numMasters=" + numMasters + ", masterClass=" + masterClass + + ", numRegionServers=" + numRegionServers + ", rsPorts=" + StringUtils.join(rsPorts) + + ", rsClass=" + rsClass + ", numDataNodes=" + numDataNodes + + ", dataNodeHosts=" + Arrays.toString(dataNodeHosts) + ", numZkServers=" + numZkServers + + ", createRootDir=" + createRootDir + ", createWALDir=" + createWALDir + '}'; + } + + /** + * @return a new builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder pattern for creating an {@link StartMiniClusterOption}. + * + * The default values of its fields should be considered public and constant. Changing the default + * values may cause other tests fail. + */ + public static final class Builder { + private int numMasters = 1; + private Class masterClass = null; + private int numRegionServers = 1; + private List rsPorts = null; + private Class rsClass = null; + private int numDataNodes = 1; + private String[] dataNodeHosts = null; + private int numZkServers = 1; + private boolean createRootDir = false; + private boolean createWALDir = false; + + private Builder() { + } + + public StartMiniClusterOption build() { + if (dataNodeHosts != null && dataNodeHosts.length != 0) { + numDataNodes = dataNodeHosts.length; + } + return new StartMiniClusterOption(numMasters, masterClass, numRegionServers, rsPorts, rsClass, + numDataNodes, dataNodeHosts, numZkServers, createRootDir, createWALDir); + } + + public Builder numMasters(int numMasters) { + this.numMasters = numMasters; + return this; + } + + public Builder masterClass(Class masterClass) { + this.masterClass = masterClass; + return this; + } + + public Builder numRegionServers(int numRegionServers) { + this.numRegionServers = numRegionServers; + return this; + } + + public Builder rsPorts(List rsPorts) { + this.rsPorts = rsPorts; + return this; + } + + public Builder rsClass(Class rsClass) { + this.rsClass = rsClass; + return this; + } + + public Builder numDataNodes(int numDataNodes) { + this.numDataNodes = numDataNodes; + return this; + } + + public Builder dataNodeHosts(String[] dataNodeHosts) { + this.dataNodeHosts = dataNodeHosts; + return this; + } + + public Builder numZkServers(int numZkServers) { + this.numZkServers = numZkServers; + return this; + } + + public Builder createRootDir(boolean createRootDir) { + this.createRootDir = createRootDir; + return this; + } + + public Builder createWALDir(boolean createWALDir) { + this.createWALDir = createWALDir; + return this; + } + } + +} -- 2.18.0