From 390a6015d80af1d617f006233867c329d54b2558 Mon Sep 17 00:00:00 2001 From: stack Date: Tue, 18 Feb 2020 21:03:15 -0800 Subject: [PATCH] HBASE-23866 More test classifications hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java Add wait on no region in transition in setup before proceeding. hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncResultScannerCursor.java Make test large. Takes longer than 50seconds. hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRSCrashPublish.java Make large because it fails when networking is weird... when VPN or something is up. See HBASE-23850. When large, doesn't get run when you do mvn test. hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableScanAll.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide2.java hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultiRowRangeFilter.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java Make it large. Takes too long. hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSeparateClientZKCluster.java Use data dir instead of /tmp. M hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestInfoServersACL.java Wait till mbean is up before proceeding w/ test. hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java Test for null CLUSTER in tearDown. --- .../client/TestAsyncMetaRegionLocator.java | 5 +- .../client/TestAsyncResultScannerCursor.java | 6 +- .../client/TestAsyncTableRSCrashPublish.java | 68 ++++++++++++------- .../hbase/client/TestAsyncTableScanAll.java | 6 +- .../client/TestScannersFromClientSide2.java | 6 +- .../client/TestSeparateClientZKCluster.java | 5 +- .../hbase/filter/TestMultiRowRangeFilter.java | 6 +- .../hadoop/hbase/http/TestInfoServersACL.java | 20 ++++++ ...stStochasticLoadBalancerRegionReplica.java | 6 +- .../security/token/SecureTestCluster.java | 2 +- 10 files changed, 84 insertions(+), 46 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java index 1f0d40b5f4..a2280439a3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionReplicaTestHelper.Locator; -import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.AfterClass; @@ -51,9 +50,9 @@ public class TestAsyncMetaRegionLocator { @BeforeClass public static void setUp() throws Exception { - TEST_UTIL.getConfiguration().set(BaseLoadBalancer.TABLES_ON_MASTER, "none"); TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3); TEST_UTIL.startMiniCluster(3); + TEST_UTIL.waitUntilNoRegionsInTransition(); REGISTRY = AsyncRegistryFactory.getRegistry(TEST_UTIL.getConfiguration()); RegionReplicaTestHelper .waitUntilAllMetaReplicasHavingRegionLocation(TEST_UTIL.getConfiguration(), REGISTRY, 3); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncResultScannerCursor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncResultScannerCursor.java index 638ae8cef0..d8e293ce68 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncResultScannerCursor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncResultScannerCursor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,12 +20,12 @@ package org.apache.hadoop.hbase.client; import java.util.concurrent.ForkJoinPool; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.experimental.categories.Category; -@Category({ MediumTests.class, ClientTests.class }) +@Category({ LargeTests.class, ClientTests.class }) public class TestAsyncResultScannerCursor extends AbstractTestResultScannerCursor { @ClassRule diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRSCrashPublish.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRSCrashPublish.java index 849feb8f2a..461bf1b9e0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRSCrashPublish.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableRSCrashPublish.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,15 +18,16 @@ package org.apache.hadoop.hbase.client; import static org.junit.Assert.assertNotEquals; - import java.io.IOException; +import java.util.concurrent.ExecutionException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -34,9 +35,10 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.apache.hbase.thirdparty.com.google.common.io.Closeables; - -@Category({ MediumTests.class, ClientTests.class }) +// Categorized as a large test so not run as part of general 'test' suite (which is small +// and mediums). This test fails if networking is odd -- say if you are connected to a +// VPN... See HBASE-23850 +@Category({ LargeTests.class, ClientTests.class }) public class TestAsyncTableRSCrashPublish { @ClassRule @@ -45,40 +47,56 @@ public class TestAsyncTableRSCrashPublish { private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static AsyncConnectionImpl CONN; - private static TableName TABLE_NAME = TableName.valueOf("Publish"); private static byte[] FAMILY = Bytes.toBytes("family"); @BeforeClass - public static void setUp() throws Exception { + public static void beforeClass() throws Exception { UTIL.getConfiguration().setBoolean(HConstants.STATUS_PUBLISHED, true); + /* Below is code for choosing a NetworkInterface and then setting it into + configs so can be picked up by the client and server. + String niName = UTIL.getConfiguration().get(HConstants.STATUS_MULTICAST_NI_NAME); + NetworkInterface ni; + if (niName != null) { + ni = NetworkInterface.getByName(niName); + } else { + String mcAddress = UTIL.getConfiguration().get(HConstants.STATUS_MULTICAST_ADDRESS, + HConstants.DEFAULT_STATUS_MULTICAST_ADDRESS); + InetAddress ina = InetAddress.getByName(mcAddress); + boolean inet6Address = ina instanceof Inet6Address; + ni = NetworkInterface.getByInetAddress(inet6Address? + Addressing.getIp6Address(): Addressing.getIp4Address()); + } + UTIL.getConfiguration().set(HConstants.STATUS_MULTICAST_NI_NAME, ni.getName()); + */ UTIL.startMiniCluster(2); UTIL.createTable(TABLE_NAME, FAMILY); UTIL.waitTableAvailable(TABLE_NAME); - CONN = - (AsyncConnectionImpl) ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get(); } @AfterClass - public static void tearDown() throws Exception { - Closeables.close(CONN, true); + public static void afterClass() throws Exception { UTIL.shutdownMiniCluster(); } @Test - public void test() throws IOException { - AsyncNonMetaRegionLocator locator = CONN.getLocator().getNonMetaRegionLocator(); - CONN.getTable(TABLE_NAME).get(new Get(Bytes.toBytes(0))).join(); - ServerName serverName = locator.getRegionLocationInCache(TABLE_NAME, HConstants.EMPTY_START_ROW) - .getDefaultRegionLocation().getServerName(); - UTIL.getMiniHBaseCluster().stopRegionServer(serverName); - UTIL.waitFor(60000, - () -> locator.getRegionLocationInCache(TABLE_NAME, HConstants.EMPTY_START_ROW) == null); - CONN.getTable(TABLE_NAME).get(new Get(Bytes.toBytes(0))).join(); - assertNotEquals(serverName, - locator.getRegionLocationInCache(TABLE_NAME, HConstants.EMPTY_START_ROW) - .getDefaultRegionLocation().getServerName()); + public void test() throws IOException, ExecutionException, InterruptedException { + Configuration conf = UTIL.getHBaseCluster().getMaster().getConfiguration(); + try (AsyncConnection connection = ConnectionFactory.createAsyncConnection(conf).get()) { + AsyncNonMetaRegionLocator locator = + ((AsyncConnectionImpl) connection).getLocator().getNonMetaRegionLocator(); + connection.getTable(TABLE_NAME).get(new Get(Bytes.toBytes(0))).join(); + ServerName serverName = + locator.getRegionLocationInCache(TABLE_NAME, HConstants.EMPTY_START_ROW) + .getDefaultRegionLocation().getServerName(); + UTIL.getMiniHBaseCluster().stopRegionServer(serverName); + UTIL.waitFor(60000, + () -> locator.getRegionLocationInCache(TABLE_NAME, HConstants.EMPTY_START_ROW) == null); + connection.getTable(TABLE_NAME).get(new Get(Bytes.toBytes(0))).join(); + assertNotEquals(serverName, + locator.getRegionLocationInCache(TABLE_NAME, HConstants.EMPTY_START_ROW) + .getDefaultRegionLocation().getServerName()); + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableScanAll.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableScanAll.java index b8adbceb50..d9a53952ab 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableScanAll.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableScanAll.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,7 @@ import java.util.List; import java.util.function.Supplier; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.ClassRule; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -30,7 +30,7 @@ import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) -@Category({ MediumTests.class, ClientTests.class }) +@Category({ LargeTests.class, ClientTests.class }) public class TestAsyncTableScanAll extends AbstractTestAsyncTableScan { @ClassRule diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide2.java index 0a10818de3..d973051c05 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide2.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -45,7 +45,7 @@ import org.junit.runners.Parameterized.Parameters; * inclusive/exclusive, limit for rows, etc. */ @RunWith(Parameterized.class) -@Category({ MediumTests.class, ClientTests.class }) +@Category({ LargeTests.class, ClientTests.class }) public class TestScannersFromClientSide2 { @ClassRule diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSeparateClientZKCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSeparateClientZKCluster.java index f44ef288b9..816801c0a8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSeparateClientZKCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSeparateClientZKCluster.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -48,7 +48,8 @@ import org.slf4j.LoggerFactory; public class TestSeparateClientZKCluster { private static final Logger LOG = LoggerFactory.getLogger(TestSeparateClientZKCluster.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final File clientZkDir = new File("/tmp/TestSeparateClientZKCluster"); + private static final File clientZkDir = + new File(TEST_UTIL.getDataTestDir("TestSeparateClientZKCluster").toString()); private static final int ZK_SESSION_TIMEOUT = 5000; private static MiniZooKeeperCluster clientZkCluster; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultiRowRangeFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultiRowRangeFilter.java index 00c557c377..9f8cca28f3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultiRowRangeFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestMultiRowRangeFilter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.Assert; @@ -48,7 +48,7 @@ import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(MediumTests.class) +@Category(LargeTests.class) public class TestMultiRowRangeFilter { @ClassRule diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestInfoServersACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestInfoServersACL.java index f605792990..7f07fbd2c3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestInfoServersACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestInfoServersACL.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.File; +import java.lang.management.ManagementFactory; import java.net.HttpURLConnection; import java.net.URL; import java.security.PrivilegedExceptionAction; @@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.LocalHBaseCluster; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.token.TokenProvider; @@ -39,6 +41,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; import org.apache.http.auth.AuthSchemeProvider; @@ -68,6 +71,10 @@ import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanInfo; +import javax.management.ObjectName; + /** * Testing info servers for admin acl. */ @@ -299,6 +306,19 @@ public class TestInfoServersACL { @Test public void testJmxAvailableForAdmins() throws Exception { final String expectedAuthorizedContent = "Hadoop:service=HBase"; + UTIL.waitFor(30000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + for (ObjectName name: ManagementFactory.getPlatformMBeanServer(). + queryNames(new ObjectName("*:*"), null)) { + if (name.toString().contains(expectedAuthorizedContent)) { + LOG.info("{}", name); + return true; + } + } + return false; + } + }); UserGroupInformation admin = UserGroupInformation.loginUserFromKeytabAndReturnUGI( USER_ADMIN_STR, KEYTAB_FILE.getAbsolutePath()); admin.doAs(new PrivilegedExceptionAction() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java index 5b06f86303..ae1371ab0a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplica.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -35,13 +35,13 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category({ MasterTests.class, MediumTests.class }) +@Category({ MasterTests.class, LargeTests.class }) public class TestStochasticLoadBalancerRegionReplica extends BalancerTestBase { @ClassRule diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java index 2263bde165..964ef140f2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java @@ -80,8 +80,8 @@ public class SecureTestCluster { public static void tearDown() throws Exception { if (CLUSTER != null) { CLUSTER.shutdown(); + CLUSTER.join(); } - CLUSTER.join(); if (KDC != null) { KDC.stop(); } -- 2.19.1