From d71fd5a240dbe7b3dc03f035e7fe3ccb5c0eacf4 Mon Sep 17 00:00:00 2001 From: Artem Ervits Date: Wed, 28 Nov 2018 10:06:31 -0500 Subject: [PATCH] HBASE-18735 Provide a fast mechanism for shutting down mini cluster --- .../apache/hadoop/hbase/HBaseTestingUtility.java | 176 +++++++++++++-------- .../org/apache/hadoop/hbase/MiniHBaseCluster.java | 19 ++- .../hadoop/hbase/TestHBaseTestingUtility.java | 67 +++++++- 3 files changed, 182 insertions(+), 80 deletions(-) diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 655bbdbe01..8f10855539 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -366,6 +366,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { } /** + * @param c configuration * @deprecated use {@link HBaseTestingUtility#HBaseTestingUtility(Configuration)} instead * @return a normal HBaseTestingUtility */ @@ -375,7 +376,9 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { } /** - * Close both the region {@code r} and it's underlying WAL. For use in tests. + * Close both the region {@code r} and it's underlying WAL.For use in tests. + * @param r region + * @throws java.io.IOException throws an exception in case of IO error. */ public static void closeRegionAndWAL(final Region r) throws IOException { closeRegionAndWAL((HRegion)r); @@ -383,16 +386,22 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * Close both the HRegion {@code r} and it's underlying WAL. For use in tests. + * @param r region + * @throws java.io.IOException throws an exception in case of IO error. */ public static void closeRegionAndWAL(final HRegion r) throws IOException { - if (r == null) return; + if (r == null) { + return; + } r.close(); - if (r.getWAL() == null) return; + if (r.getWAL() == null) { + return; + } r.getWAL().close(); } /** - * Returns this classes's instance of {@link Configuration}. Be careful how + * Returns this class's instance of {@link Configuration}. Be careful how * you use the returned Configuration since {@link Connection} instances * can be shared. The Map of Connections is keyed by the Configuration. If * say, a Connection was being used against a cluster that had been shutdown, @@ -464,11 +473,11 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { sysValue + " so I do NOT create it in " + parent); String confValue = conf.get(propertyName); if (confValue != null && !confValue.endsWith(sysValue)){ - LOG.warn( - propertyName + " property value differs in configuration and system: "+ - "Configuration="+confValue+" while System="+sysValue+ - " Erasing configuration value by system value." - ); + LOG.warn( + propertyName + " property value differs in configuration and system: "+ + "Configuration="+confValue+" while System="+sysValue+ + " Erasing configuration value by system value." + ); } conf.set(propertyName, sysValue); } else { @@ -480,7 +489,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * @return Where to write test data on the test filesystem; Returns working directory - * for the test filesystem by default + for the test filesystem by default * @see #setupDataTestDirOnTestFS() * @see #getTestFileSystem() */ @@ -562,11 +571,15 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) { newDataTestDir = new Path(getDataTestDir(), randomStr); File dataTestDir = new File(newDataTestDir.toString()); - if (deleteOnExit()) dataTestDir.deleteOnExit(); + if (deleteOnExit()) { + dataTestDir.deleteOnExit(); + } } else { Path base = getBaseTestDirOnTestFS(); newDataTestDir = new Path(base, randomStr); - if (deleteOnExit()) fs.deleteOnExit(newDataTestDir); + if (deleteOnExit()) { + fs.deleteOnExit(newDataTestDir); + } } return newDataTestDir; } @@ -578,8 +591,9 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { */ public boolean cleanupDataTestDirOnTestFS() throws IOException { boolean ret = getTestFileSystem().delete(dataTestDirOnTestFS, true); - if (ret) + if (ret) { dataTestDirOnTestFS = null; + } return ret; } @@ -615,9 +629,9 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @see #shutdownMiniDFSCluster() * @return The mini dfs cluster created. */ - public MiniDFSCluster startMiniDFSCluster(final String hosts[]) - throws Exception { - if ( hosts != null && hosts.length != 0) { + public MiniDFSCluster startMiniDFSCluster(final String [] hosts) + throws Exception { + if (hosts != null && hosts.length != 0) { return startMiniDFSCluster(hosts.length, hosts); } else { return startMiniDFSCluster(1, null); @@ -629,12 +643,13 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * Can only create one. * @param servers How many DNs to start. * @param hosts hostnames DNs to run on. - * @throws Exception + * @throws Exception in case MiniDFS cannot be started * @see #shutdownMiniDFSCluster() * @return The mini dfs cluster created. */ - public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[]) - throws Exception { + public MiniDFSCluster startMiniDFSCluster(int servers, final String [] hosts) + throws Exception + { return startMiniDFSCluster(servers, null, hosts); } @@ -795,7 +810,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { */ @Deprecated public MiniHBaseCluster startMiniCluster(int numSlaves, boolean createRootDir) - throws Exception { + throws Exception { StartMiniClusterOption option = StartMiniClusterOption.builder() .numRegionServers(numSlaves).numDataNodes(numSlaves).createRootDir(createRootDir).build(); return startMiniCluster(option); @@ -1209,7 +1224,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * @return Current mini hbase cluster. Only has something in it after a call - * to {@link #startMiniCluster()}. + to {@link #startMiniCluster()}. * @see #startMiniCluster() */ public MiniHBaseCluster getMiniHBaseCluster() { @@ -1238,8 +1253,40 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. + * @throws java.io.IOException may throw due to IO issues */ public void shutdownMiniHBaseCluster() throws IOException { + cleanup(); + if (this.hbaseCluster != null) { + this.hbaseCluster.shutdown(); + // Wait till hbase is down before going on to shutdown zk. + this.hbaseCluster.waitUntilShutDown(); + this.hbaseCluster = null; + } + if (zooKeeperWatcher != null) { + zooKeeperWatcher.close(); + zooKeeperWatcher = null; + } + } + + /** + * Abruptly Shutdown HBase mini cluster.Does not shutdown zk or dfs if running. + * @throws java.io.IOException may throw due to IO issues + */ + public void killMiniHBaseCluster() throws IOException { + cleanup(); + if (this.hbaseCluster != null) { + getMiniHBaseCluster().killAll(); + this.hbaseCluster = null; + } + if (zooKeeperWatcher != null) { + zooKeeperWatcher.close(); + zooKeeperWatcher = null; + } + } + + // close hbase admin, close current connection and reset MIN MAX configs for RS. + private void cleanup() throws IOException { if (hbaseAdmin != null) { hbaseAdmin.close(); hbaseAdmin = null; @@ -1251,23 +1298,14 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { // unset the configuration for MIN and MAX RS to start conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); - if (this.hbaseCluster != null) { - this.hbaseCluster.shutdown(); - // Wait till hbase is down before going on to shutdown zk. - this.hbaseCluster.waitUntilShutDown(); - this.hbaseCluster = null; - } - if (zooKeeperWatcher != null) { - zooKeeperWatcher.close(); - zooKeeperWatcher = null; - } } /** - * Returns the path to the default root dir the minicluster uses. If create - * is true, a new root directory path is fetched irrespective of whether it has been fetched - * before or not. If false, previous path is used. - * Note: this does not cause the root dir to be created. + * Returns the path to the default root dir the minicluster uses. + * @param create If create is true, a new root directory path is + fetched irrespective of whether it has been fetched + before or not. If false, previous path is used. + Note: this does not cause the root dir to be created. * @return Fully qualified path for the default hbase root dir * @throws IOException */ @@ -1296,9 +1334,9 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * is created for you as part of mini cluster startup. You'd only use this * method if you were doing manual operation. * @param create This flag decides whether to get a new - * root or data directory path or not, if it has been fetched already. - * Note : Directory will be made irrespective of whether path has been fetched or not. - * If directory already exists, it will be overwritten + root or data directory path or not, if it has been fetched already. + Note : Directory will be made irrespective of whether path has been fetched or not. + If directory already exists, it will be overwritten * @return Fully qualified path to hbase root dir * @throws IOException */ @@ -1388,7 +1426,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Table createTable(TableName tableName, String family) - throws IOException{ + throws IOException{ return createTable(tableName, new String[]{family}); } @@ -1400,7 +1438,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Table createTable(TableName tableName, String[] families) - throws IOException { + throws IOException { List fams = new ArrayList<>(families.length); for (String family : families) { fams.add(Bytes.toBytes(family)); @@ -1416,7 +1454,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Table createTable(TableName tableName, byte[] family) - throws IOException{ + throws IOException{ return createTable(tableName, new byte[][]{family}); } @@ -1429,7 +1467,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Table createMultiRegionTable(TableName tableName, byte[] family, int numRegions) - throws IOException { + throws IOException { if (numRegions < 3) throw new IOException("Must create at least 3 regions"); byte[] startKey = Bytes.toBytes("aaaaa"); byte[] endKey = Bytes.toBytes("zzzzz"); @@ -1446,7 +1484,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Table createTable(TableName tableName, byte[][] families) - throws IOException { + throws IOException { return createTable(tableName, families, (byte[][]) null); } @@ -1491,7 +1529,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { public Table createTable(TableName tableName, byte[][] families, int numVersions, byte[] startKey, byte[] endKey, int numRegions) - throws IOException{ + throws IOException{ HTableDescriptor desc = createTableDescriptor(tableName, families, numVersions); getAdmin().createTable(desc, startKey, endKey, numRegions); @@ -1510,7 +1548,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Table createTable(TableDescriptor htd, byte[][] families, Configuration c) - throws IOException { + throws IOException { return createTable(htd, families, null, c); } @@ -1601,7 +1639,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public Table createTable(TableName tableName, byte[] family, int numVersions) - throws IOException { + throws IOException { return createTable(tableName, new byte[][]{family}, numVersions); } @@ -1681,22 +1719,22 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { public Table createTable(TableName tableName, byte[][] families, int numVersions, int blockSize, String cpName) throws IOException { - HTableDescriptor desc = new HTableDescriptor(tableName); - for (byte[] family : families) { - HColumnDescriptor hcd = new HColumnDescriptor(family) - .setMaxVersions(numVersions) - .setBlocksize(blockSize); - desc.addFamily(hcd); - } - if(cpName != null) { - desc.addCoprocessor(cpName); - } - getAdmin().createTable(desc); - // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are - // assigned - waitUntilAllRegionsAssigned(tableName); - return getConnection().getTable(tableName); + HTableDescriptor desc = new HTableDescriptor(tableName); + for (byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family) + .setMaxVersions(numVersions) + .setBlocksize(blockSize); + desc.addFamily(hcd); + } + if(cpName != null) { + desc.addCoprocessor(cpName); } + getAdmin().createTable(desc); + // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are + // assigned + waitUntilAllRegionsAssigned(tableName); + return getConnection().getTable(tableName); + } /** * Create a table. @@ -1708,7 +1746,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { */ public Table createTable(TableName tableName, byte[][] families, int[] numVersions) - throws IOException { + throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); int i = 0; for (byte[] family : families) { @@ -1763,9 +1801,9 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { throws IOException, InterruptedException { admin.modifyTable(desc); Pair status = new Pair() {{ - setFirst(0); - setSecond(0); - }}; + setFirst(0); + setSecond(0); + }}; int i = 0; do { status = admin.getAlterStatus(desc.getTableName()); @@ -1904,7 +1942,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { */ public HRegion createLocalHRegion(TableDescriptor desc, byte [] startKey, byte [] endKey) - throws IOException { + throws IOException { HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey); return createLocalHRegion(hri, desc); } @@ -2186,7 +2224,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { * @throws IOException */ public int loadRegion(final HRegion r, final byte[] f, final boolean flush) - throws IOException { + throws IOException { byte[] k = new byte[3]; int rowCount = 0; for (byte b1 = 'a'; b1 <= 'z'; b1++) { @@ -2457,7 +2495,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { */ public List createMultiRegionsInMeta(final Configuration conf, final TableDescriptor htd, byte [][] startKeys) - throws IOException { + throws IOException { Table meta = getConnection().getTable(TableName.META_TABLE_NAME); Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR); List newRegions = new ArrayList<>(startKeys.length); @@ -2841,7 +2879,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { } public void expireSession(ZKWatcher nodeZK) throws Exception { - expireSession(nodeZK, false); + expireSession(nodeZK, false); } /** diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index 473eb74abf..15a688bcfa 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -49,7 +49,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto /** * This class creates a single process HBase cluster. - * each server. The master uses the 'default' FileSystem. The RegionServers, + * The master uses the 'default' FileSystem. The RegionServers, * if we are running on DistributedFilesystem, create a FileSystem instance * each and will close down their instance on the way out. */ @@ -87,6 +87,8 @@ public class MiniHBaseCluster extends HBaseCluster { * @param conf Configuration to be used for cluster * @param numMasters initial number of masters to start. * @param numRegionServers initial number of region servers to start. + * @param masterClass master class + * @param regionserverClass RegionServer class */ public MiniHBaseCluster(Configuration conf, int numMasters, int numRegionServers, Class masterClass, @@ -96,10 +98,15 @@ public class MiniHBaseCluster extends HBaseCluster { } /** + * @param conf Configuration to be used for cluster + * @param numMasters initial number of masters to start. * @param rsPorts Ports that RegionServer should use; pass ports if you want to test cluster * restart where for sure the regionservers come up on same address+port (but * just with different startcode); by default mini hbase clusters choose new * arbitrary ports on each cluster start. + * @param numRegionServers initial number of region servers to start. + * @param masterClass master class + * @param regionserverClass RegionServer class * @throws IOException * @throws InterruptedException */ @@ -419,10 +426,11 @@ public class MiniHBaseCluster extends HBaseCluster { } /** - * Starts a region server thread and waits until its processed by master. Throws an exception + * Starts a region server thread and waits until its processed by master.Throws an exception * when it can't start a region server or when the region server is not processed by master * within the timeout. * + * @param timeout timeout for waiting * @return New RegionServerThread */ public JVMClusterUtil.RegionServerThread startRegionServerAndWait(long timeout) @@ -449,6 +457,7 @@ public class MiniHBaseCluster extends HBaseCluster { /** * Cause a region server to exit doing basic clean up only on its way out. * @param serverNumber Used as index into a list. + * @return server regionserver to abort */ public String abortRegionServer(int serverNumber) { HRegionServer server = getRegionServer(serverNumber); @@ -558,6 +567,7 @@ public class MiniHBaseCluster extends HBaseCluster { /** * Cause a master to exit without shutting down entire cluster. * @param serverNumber Used as index into a list. + * @return server master to abort */ public String abortMaster(int serverNumber) { HMaster server = getMaster(serverNumber); @@ -611,7 +621,6 @@ public class MiniHBaseCluster extends HBaseCluster { * * @return true if an active master becomes available. false if there are no * masters left. - * @throws InterruptedException */ @Override public boolean waitForActiveAndReadyMaster(long timeout) throws IOException { @@ -705,6 +714,7 @@ public class MiniHBaseCluster extends HBaseCluster { /** * Call flushCache on all regions of the specified table. + * @param tableName table to flushCache on. */ public void flushcache(TableName tableName) throws IOException { for (JVMClusterUtil.RegionServerThread t : this.hbaseCluster.getRegionServers()) { @@ -718,6 +728,7 @@ public class MiniHBaseCluster extends HBaseCluster { /** * Call flushCache on all regions on all participating regionservers. + * @param major compaction * @throws IOException */ public void compact(boolean major) throws IOException { @@ -731,6 +742,8 @@ public class MiniHBaseCluster extends HBaseCluster { /** * Call flushCache on all regions of the specified table. + * @param tableName table to compact + * @param major compaction * @throws IOException */ public void compact(TableName tableName, boolean major) throws IOException { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java index 97de8a9e77..a4e6516c65 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java @@ -26,11 +26,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.File; -import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -62,6 +58,9 @@ import org.slf4j.LoggerFactory; */ @Category({MiscTests.class, LargeTests.class}) public class TestHBaseTestingUtility { + private static final int NUMTABLES = 100; + private static final int NUMROWS = 1000; + private static final int NUMREGIONS = 10; @ClassRule public static final HBaseClassTestRule CLASS_RULE = @@ -76,7 +75,7 @@ public class TestHBaseTestingUtility { * Basic sanity test that spins up multiple HDFS and HBase clusters that share * the same ZK ensemble. We then create the same table in both and make sure * that what we insert in one place doesn't end up in the other. - * @throws Exception + * @throws Exception throws exception in case creation of clusters fails. */ @Test public void testMultiClusters() throws Exception { @@ -199,13 +198,13 @@ public class TestHBaseTestingUtility { htu1.startMiniCluster(); htu1.getDFSCluster().getFileSystem().create(foo); - assertTrue( htu1.getDFSCluster().getFileSystem().exists(foo)); + assertTrue(htu1.getDFSCluster().getFileSystem().exists(foo)); htu1.shutdownMiniCluster(); htu1.startMiniCluster(); - assertFalse( htu1.getDFSCluster().getFileSystem().exists(foo)); + assertFalse(htu1.getDFSCluster().getFileSystem().exists(foo)); htu1.getDFSCluster().getFileSystem().create(foo); - assertTrue( htu1.getDFSCluster().getFileSystem().exists(foo)); + assertTrue(htu1.getDFSCluster().getFileSystem().exists(foo)); htu1.shutdownMiniCluster(); } @@ -471,4 +470,56 @@ public class TestHBaseTestingUtility { htu.shutdownMiniCluster(); } } + + @Test + public void testShutdownMiniHBaseCluster() throws Exception { + + HBaseTestingUtility htu = new HBaseTestingUtility(); + htu.startMiniZKCluster(); + + try { + htu.startMiniHBaseCluster(); + + TableName tableName; + byte[] FAM_NAME; + + for(int i = 0; i < NUMTABLES; i++) { + tableName = TableName.valueOf(name.getMethodName() + i); + FAM_NAME = Bytes.toBytes("fam" + i); + + try (Table table = htu.createMultiRegionTable(tableName, FAM_NAME, NUMREGIONS)) { + htu.loadRandomRows(table, FAM_NAME, 100, NUMROWS); + } + } + } finally { + htu.shutdownMiniHBaseCluster(); + htu.shutdownMiniZKCluster(); + } + } + + @Test + public void testKillMiniHBaseCluster() throws Exception { + + HBaseTestingUtility htu = new HBaseTestingUtility(); + htu.startMiniZKCluster(); + + try { + htu.startMiniHBaseCluster(); + + TableName tableName; + byte[] FAM_NAME; + + for(int i = 0; i < NUMTABLES; i++) { + tableName = TableName.valueOf(name.getMethodName() + i); + FAM_NAME = Bytes.toBytes("fam" + i); + + try (Table table = htu.createMultiRegionTable(tableName, FAM_NAME, NUMREGIONS)) { + htu.loadRandomRows(table, FAM_NAME, 100, NUMROWS); + } + } + } finally { + htu.killMiniHBaseCluster(); + htu.shutdownMiniZKCluster(); + } + } } -- 2.16.2