commit 533e857feeb762ed02f228893eae8ed5046b4db8 Author: Janaki Lahorani Date: Sat Nov 25 10:42:28 2017 -0800 HIVE-18147: Retry getting a new port number for 10 times if the starting of HMS or HS2 fails with the picked port Change-Id: Ic0c9376d5f684999d869080a1c824d020220949b diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 80c5d63e64e044af7f22f0868f5a3dbe08e8c1b8..e5aa2b8d7ad9c21fc2064cf5e35a6b1a77705258 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -80,9 +80,7 @@ protected void setUp() throws Exception { return; } - - msPort = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); + msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); isServerRunning = true; diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index a000642bf3dcbcf76fc89dfe9866f25ba0b232ad..6711a3edeca2bf748d2b7a8c7ef86decc95bdb55 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -105,9 +105,8 @@ public static void setup() throws Exception { return; } - msPort = MetaStoreTestUtils.findFreePort(); + msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); - MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); diff --git hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java index 341314b2184cad43c3b62cfea20e40d545708362..55cd5b121d03fea15c3158039ed5f58b6ab5c7c2 100644 --- hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java +++ hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java @@ -70,21 +70,38 @@ private static final String charSet = "UTF-8"; @BeforeClass - public static void startHebHcatInMem() { - int webhcatPort = 50111; - try { - //in case concurrent tests are running on the same machine - webhcatPort = MetaStoreTestUtils.findFreePort(); + public static void startHebHcatInMem() throws Exception { + Exception webhcatException = null; + int webhcatPort = 0; + boolean webhcatStarted = false; + + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { + try { + if (tryCount == MetaStoreTestUtils.RETRY_COUNT - 1) { + /* Last try to get a port. Just use default 50111. */ + webhcatPort = 50111; + LOG.warn("Unable to find free port; using default: " + webhcatPort); + } + else { + webhcatPort = MetaStoreTestUtils.findFreePort(); + } + templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); + templetonServer = new Main(new String[] { "-D" + AppConfig.UNIT_TEST_MODE + "=true", + "-D" + AppConfig.PORT + "=" + webhcatPort }); + LOG.info("Starting Main; WebHCat using port: " + webhcatPort); + templetonServer.run(); + LOG.info("Main started"); + webhcatStarted = true; + break; + } catch (Exception ce) { + LOG.info("Attempt to Start WebHCat using port: " + webhcatPort + " failed"); + webhcatException = ce; + } } - catch (IOException ex) { - LOG.warn("Unable to find free port; using default: " + webhcatPort); + + if (!webhcatStarted) { + throw webhcatException; } - templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); - templetonServer = new Main(new String[] {"-D" + - AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort}); - LOG.info("Starting Main; WebHCat using port: " + webhcatPort); - templetonServer.run(); - LOG.info("Main started"); } @AfterClass diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java index ebdfab2b5f89abf9e3695912e27884ba4cfac9aa..bd8332c49c3450de376cfda743c013618b4162b7 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java @@ -57,8 +57,7 @@ protected static void setup() throws Exception { hiveConf = new HiveConf(); if (isRemoteMetastoreMode) { - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); } hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 6a2bbd2841ec06bc5cd83142bb68eeb8703741f5..8366fabf8a7da3cd9cc7aff63703568fc7cab3d7 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -174,9 +174,8 @@ public static void setUp() throws Exception { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName()); UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName()); - int port = MetaStoreTestUtils.findFreePort(); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); SessionState.start(new CliSessionState(hiveConf)); msc = new HiveMetaStoreClient(hiveConf); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java index 3fa4b1eb28ec77b39854c96e26bf8b24e49de1c9..5267dcaa5fe855a838d601d3c4051dfb2aa3c874 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java @@ -73,8 +73,7 @@ protected void setUp() throws Exception { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java index 27bc791c1f001a45f81e3ad8d7bd17dc01b53cb1..3bb340e157c5a29d970d38c54d514d5c747ef6db 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java @@ -48,8 +48,7 @@ protected void setUp() throws Exception { DummyPreListener.class.getName()); System.setProperty("hive.metastore.end.function.listeners", DummyEndFunctionListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 4df2f8661c09a9a46c567c0931ad60d71b5c6e1e..72197745601ae8edb1d9387e88562c2a6ddd013c 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -94,11 +94,10 @@ protected void setUp() throws Exception { System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN, metaConfVal); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java index d9da42229d0492100d301359253cb05d23f0c840..a9053241cc128862f1e68ac6eff4f633ff77a7f1 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java @@ -50,8 +50,7 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname, DummyRawStoreControlledCommit.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java index 9578ef9219bfde1c3e915529938a72e24d7a3c14..3d58819b1a203fd9dae48d6950b559602d457c5a 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java @@ -41,8 +41,7 @@ protected void setUp() throws Exception { super.setUp(); System.setProperty("hive.metastore.init.hooks", DummyMetaStoreInitListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java index b4d258ca3e3dc8abdf430e770a0ded22fbe409a6..a8b0642ae709a6baee51f5637d5a3df440970cf1 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java @@ -33,9 +33,8 @@ public void testInitListenerException() throws Throwable { System.setProperty("hive.metastore.init.hooks", ErrorInitListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); try { - HiveMetaStore.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStoreWithRetry(); } catch (Throwable throwable) { Assert.assertEquals(MetaException.class, throwable.getClass()); Assert.assertEquals( @@ -50,9 +49,8 @@ public void testEventListenerException() throws Throwable { System.setProperty("hive.metastore.init.hooks", ""); System.setProperty("hive.metastore.event.listeners", ErrorEventListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); try { - HiveMetaStore.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStoreWithRetry(); } catch (Throwable throwable) { Assert.assertEquals(MetaException.class, throwable.getClass()); Assert.assertEquals( diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index a899739f47b818b5da5a57d2af02ffbaa7668f09..9d241a89c511e2672c0a740afff8fe6e0cf16221 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -40,10 +40,7 @@ @BeforeClass public static void before() throws Exception { - int port = MetaStoreTestUtils.findFreePort(); - hiveConf = new HiveConf(TestMetaStoreMetrics.class); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); @@ -52,7 +49,9 @@ public static void before() throws Exception { "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); //Increments one HMS connection - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); + + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); //Increments one HMS connection (Hive.get()) SessionState.start(new CliSessionState(hiveConf)); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index 19bed9d0977aea7af9af19405fcf2acd79e07610..d4a809f1428763bea748adfaaef99db1f13e66be 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -42,9 +42,8 @@ protected void setUp() throws Exception { return; } - port = MetaStoreTestUtils.findFreePort(); + port = MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); System.out.println("Starting MetaStore Server on port " + port); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); isServerStarted = true; // This is default case with setugi off for both client and server diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java index 449fe71a05015edd832ffd12d4d74615de86448a..9ba1d75303d91e1d8842ae35a4301aa45c20b677 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -48,11 +48,10 @@ protected void setUp() throws Exception { return; } - int port = MetaStoreTestUtils.findFreePort(); - System.out.println("Starting MetaStore Server on port " + port); System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname, IpAddressListener.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); + System.out.println("Started MetaStore Server on port " + port); isServerStarted = true; // This is default case with setugi off for both client and server diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java index 41494d183e564c6294b4db664a52e40f775c80b3..2b7c5a12139d40c44dfc355e1aaff6cd34714bfa 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -53,8 +53,7 @@ protected void setUp() throws Exception { super.setUp(); System.setProperty("hive.metastore.pre.event.listeners", AlternateFailurePreListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index 4ce27311d128f75eb44e33380d831c64ffa9aea4..179eed95d0be28056d0bb7ed6a8b63910a7594ac 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -80,8 +80,7 @@ protected void setUp() throws Exception { SessionState.start(conf); // Test with remote metastore service - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java index fa5ae809a810f3bcd79db076af919ffe83522b9e..dc3af3c18696bf7ce14b6139e4686c340b3fff32 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java @@ -62,9 +62,6 @@ protected HiveConf createHiveConf() throws Exception { @Before public void setUp() throws Exception { - - int port = MetaStoreTestUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -74,7 +71,7 @@ public void setUp() throws Exception { InjectableDummyAuthenticator.class.getName()); clientHiveConf = createHiveConf(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), clientHiveConf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(clientHiveConf); // Turn off client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java index 102af6a08a5eb4336b303eacae0159b1ecd23758..6a668aa40c856f90d09da4eb8941a0e7e0a70f19 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java @@ -53,8 +53,6 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname, @@ -62,7 +60,7 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java index 60273e7aad22b61c9963cd52186445d93ee7b947..57ff8c9ae7edc71b12a46ce63387e0ebdef58b22 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java @@ -58,13 +58,11 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - // Turn off metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, ""); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java index f0196c605c65b8d818d312fb7be673e7a869501a..871cf3db764421a2e91f3c2dce30c0c4b87209d4 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java @@ -84,8 +84,6 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -98,8 +96,7 @@ protected void setUp() throws Exception { InjectableDummyAuthenticator.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, ""); - - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = createHiveConf(); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java index 62835b090ca52439ef5e737355017554a3ede8d3..2059370fd40c49ca0016ee8179512d325768331b 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java @@ -47,10 +47,6 @@ @BeforeClass public static void setUp() throws Exception { - - - int port = MetaStoreTestUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -62,7 +58,7 @@ public static void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(); diff --git itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index e554cc83e513b9a40183510fbc1338c52c58c1f0..273cd0176a2ff01604fbffc0c3668bce5a205e8e 100644 --- itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -75,6 +75,7 @@ private final boolean cleanupLocalDirOnStartup; private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; + private boolean usePortsFromConf = false; public enum MiniClusterType { MR, @@ -227,6 +228,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM this.isMetastoreRemote = isMetastoreRemote; this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; + this.usePortsFromConf = usePortsFromConf; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); FileSystem fs; @@ -334,18 +336,42 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsF public void start(Map confOverlay) throws Exception { if (isMetastoreRemote) { - int metaStorePort = MetaStoreTestUtils.findFreePort(); + int metaStorePort = MetaStoreTestUtils.startMetaStoreWithRetry(getHiveConf()); getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); - MetaStoreTestUtils.startMetaStore(metaStorePort, HadoopThriftAuthBridge.getBridge(), getHiveConf()); } - hiveServer2 = new HiveServer2(); // Set confOverlay parameters for (Map.Entry entry : confOverlay.entrySet()) { setConfProperty(entry.getKey(), entry.getValue()); } - hiveServer2.init(getHiveConf()); - hiveServer2.start(); + + Exception HS2Exception = null; + boolean HS2Started = false; + for (int tryCount = 0; (tryCount < MetaStoreTestUtils.RETRY_COUNT); tryCount++) { + try { + hiveServer2 = new HiveServer2(); + hiveServer2.init(getHiveConf()); + hiveServer2.start(); + HS2Started = true; + break; + } catch (Exception t) { + HS2Exception = t; + if (usePortsFromConf) { + HS2Started = false; + break; + } else { + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, + MetaStoreTestUtils.findFreePort()); + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, + MetaStoreTestUtils.findFreePort()); + } + } + } + + if (!HS2Started) { + throw(HS2Exception); + } + waitForStartup(); setStarted(true); } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java index 28353c3780f82bc69af28e7e56da535e257f9871..7df7ac55b83ab10c0c375216e1b3d06916f1e230 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; import java.io.IOException; +import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -35,6 +36,7 @@ public class MetaStoreTestUtils { private static final Logger LOG = LoggerFactory.getLogger("hive.log"); + public static final int RETRY_COUNT = 10; public static int startMetaStore() throws Exception { return MetaStoreTestUtils.startMetaStore(HadoopThriftAuthBridge.getBridge(), null); @@ -76,6 +78,36 @@ public void run() { MetaStoreTestUtils.loopUntilHMSReady(port); } + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge) throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(bridge, null); + } + + public static int startMetaStoreWithRetry(HiveConf conf) throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); + } + + public static int startMetaStoreWithRetry() throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), null); + } + + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge, HiveConf conf) + throws Exception { + Exception metaStoreException = null; + int metaStorePort = 0; + + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { + try { + metaStorePort = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(metaStorePort, bridge, conf); + return metaStorePort; + } catch (ConnectException ce) { + metaStoreException = ce; + } + } + + throw metaStoreException; + } + /** * A simple connect test to make sure that the metastore is up * @throws Exception diff --git service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java index 6660097672d7e6f8ad2477ff03d54a05a1dbfeac..1dac7833213ab02b326e7ff4df407b6f4808a353 100644 --- service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java +++ service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java @@ -96,15 +96,27 @@ public static void tearDownAfterClass() throws Exception { } protected static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception { - hiveServer2.init(hiveConf); - // Start HiveServer2 with given config - // Fail if server doesn't start - try { - hiveServer2.start(); - } catch (Throwable t) { - t.printStackTrace(); + Exception HS2Exception = null; + boolean HS2Started = false; + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2.init(hiveConf); + hiveServer2.start(); + HS2Started = true; + break; + } catch (Exception t) { + HS2Exception = t; + port = MetaStoreTestUtils.findFreePort(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, port); + hiveServer2 = new HiveServer2(); + } + } + + if (!HS2Started) { + HS2Exception.printStackTrace(); fail(); } + // Wait for startup to complete Thread.sleep(2000); System.out.println("HiveServer2 started on port " + port);