commit 16db137d94839c6c99814765d2df02e7675911d7 Author: Janaki Lahorani Date: Sat Nov 25 10:42:28 2017 -0800 HIVE-18147: Retry getting a new port number for 10 times if the starting of HMS or HS2 fails with the picked port Change-Id: Ic0c9376d5f684999d869080a1c824d020220949b diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 80c5d63e64e044af7f22f0868f5a3dbe08e8c1b8..7bf547c94258828a17441cdce47c3776aa303ee9 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -80,9 +80,7 @@ protected void setUp() throws Exception { return; } - - msPort = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); + msPort = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); isServerRunning = true; diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index a000642bf3dcbcf76fc89dfe9866f25ba0b232ad..9824c459f83f1d549e87067183cebd928b9430ef 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -105,9 +105,8 @@ public static void setup() throws Exception { return; } - msPort = MetaStoreTestUtils.findFreePort(); + msPort = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); - MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); diff --git hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java index 341314b2184cad43c3b62cfea20e40d545708362..ba2e2813df85fd46312c507c3509ded77727e8a9 100644 --- hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java +++ hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java @@ -70,21 +70,37 @@ private static final String charSet = "UTF-8"; @BeforeClass - public static void startHebHcatInMem() { - int webhcatPort = 50111; - try { - //in case concurrent tests are running on the same machine - webhcatPort = MetaStoreTestUtils.findFreePort(); + public static void startHebHcatInMem() throws Exception { + Exception webhcatException = null; + int webhcatPort = 0; + boolean webhcatStarted = false; + + for (int tryCount = 0; tryCount < MetaStoreTestUtils.retryCount; tryCount++) { + try { + if (tryCount == MetaStoreTestUtils.retryCount - 1) { + /* Last try to get a port. Just use default 50111. */ + webhcatPort = 50111; + LOG.warn("Unable to find free port; using default: " + webhcatPort); + } + else { + webhcatPort = MetaStoreTestUtils.findFreePort(); + } + templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); + templetonServer = new Main(new String[] { "-D" + AppConfig.UNIT_TEST_MODE + "=true", + "-D" + AppConfig.PORT + "=" + webhcatPort }); + LOG.info("Starting Main; WebHCat using port: " + webhcatPort); + templetonServer.run(); + LOG.info("Main started"); + webhcatStarted = true; + } catch (Exception ce) { + LOG.info("Attempt to Start WebHCat using port: " + webhcatPort + " failed"); + webhcatException = ce; + } } - catch (IOException ex) { - LOG.warn("Unable to find free port; using default: " + webhcatPort); + + if (!webhcatStarted) { + throw webhcatException; } - templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); - templetonServer = new Main(new String[] {"-D" + - AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort}); - LOG.info("Starting Main; WebHCat using port: " + webhcatPort); - templetonServer.run(); - LOG.info("Main started"); } @AfterClass diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java index ebdfab2b5f89abf9e3695912e27884ba4cfac9aa..b167f6cbd8a6d023a348bf8b447bf9d08cf4fa98 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java @@ -57,8 +57,7 @@ protected static void setup() throws Exception { hiveConf = new HiveConf(); if (isRemoteMetastoreMode) { - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); } hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 6a2bbd2841ec06bc5cd83142bb68eeb8703741f5..da37304f481d4823510b6982a214336f3fbf67c1 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -174,9 +174,9 @@ public static void setUp() throws Exception { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName()); UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName()); - int port = MetaStoreTestUtils.findFreePort(); + int port = + MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); SessionState.start(new CliSessionState(hiveConf)); msc = new HiveMetaStoreClient(hiveConf); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java index 3fa4b1eb28ec77b39854c96e26bf8b24e49de1c9..7e4337b37fe14ca7b3638218f0e52a9765d14e1d 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java @@ -73,8 +73,7 @@ protected void setUp() throws Exception { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java index 27bc791c1f001a45f81e3ad8d7bd17dc01b53cb1..19baf7fec0eb9092d92fdbfaaa422f728d77bddc 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java @@ -48,8 +48,7 @@ protected void setUp() throws Exception { DummyPreListener.class.getName()); System.setProperty("hive.metastore.end.function.listeners", DummyEndFunctionListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 4df2f8661c09a9a46c567c0931ad60d71b5c6e1e..79f47faccad8275473450577e246715e99fff8a2 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -94,11 +94,11 @@ protected void setUp() throws Exception { System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN, metaConfVal); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + int port = + MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java index d9da42229d0492100d301359253cb05d23f0c840..cd26f45f8a69c3a415465f31ef6708c91229fb01 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java @@ -50,8 +50,7 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname, DummyRawStoreControlledCommit.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java index 9578ef9219bfde1c3e915529938a72e24d7a3c14..2f0015a81e2528511f09c33da0221a6d2d86e718 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java @@ -41,8 +41,7 @@ protected void setUp() throws Exception { super.setUp(); System.setProperty("hive.metastore.init.hooks", DummyMetaStoreInitListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java index b4d258ca3e3dc8abdf430e770a0ded22fbe409a6..fadca24e5445f13917997803986366e40516d507 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java @@ -33,9 +33,8 @@ public void testInitListenerException() throws Throwable { System.setProperty("hive.metastore.init.hooks", ErrorInitListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); try { - HiveMetaStore.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); } catch (Throwable throwable) { Assert.assertEquals(MetaException.class, throwable.getClass()); Assert.assertEquals( @@ -50,9 +49,8 @@ public void testEventListenerException() throws Throwable { System.setProperty("hive.metastore.init.hooks", ""); System.setProperty("hive.metastore.event.listeners", ErrorEventListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); try { - HiveMetaStore.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); } catch (Throwable throwable) { Assert.assertEquals(MetaException.class, throwable.getClass()); Assert.assertEquals( diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index a899739f47b818b5da5a57d2af02ffbaa7668f09..50e8e6488196b66fb1ba968057a85645bd4b5f70 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -40,10 +40,7 @@ @BeforeClass public static void before() throws Exception { - int port = MetaStoreTestUtils.findFreePort(); - hiveConf = new HiveConf(TestMetaStoreMetrics.class); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); @@ -52,7 +49,10 @@ public static void before() throws Exception { "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); //Increments one HMS connection - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + int port = + MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), hiveConf); + + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); //Increments one HMS connection (Hive.get()) SessionState.start(new CliSessionState(hiveConf)); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index 19bed9d0977aea7af9af19405fcf2acd79e07610..0efa7ad2a77aa62acd2b63da04391973d4b7a4fa 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -42,9 +42,8 @@ protected void setUp() throws Exception { return; } - port = MetaStoreTestUtils.findFreePort(); + port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), hiveConf); System.out.println("Starting MetaStore Server on port " + port); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); isServerStarted = true; // This is default case with setugi off for both client and server diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java index 449fe71a05015edd832ffd12d4d74615de86448a..ce8ac4316205cb1d1b79e0247be623276ff430e4 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -48,11 +48,10 @@ protected void setUp() throws Exception { return; } - int port = MetaStoreTestUtils.findFreePort(); - System.out.println("Starting MetaStore Server on port " + port); System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname, IpAddressListener.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); + System.out.println("Starting MetaStore Server on port " + port); isServerStarted = true; // This is default case with setugi off for both client and server diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java index 41494d183e564c6294b4db664a52e40f775c80b3..6a61458e2f889c94127cfb3e5041d767671cb53f 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -53,8 +53,7 @@ protected void setUp() throws Exception { super.setUp(); System.setProperty("hive.metastore.pre.event.listeners", AlternateFailurePreListener.class.getName()); - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index 4ce27311d128f75eb44e33380d831c64ffa9aea4..5f43a9ee8572ef55da31417dc173450ad68c7072 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -80,8 +80,7 @@ protected void setUp() throws Exception { SessionState.start(conf); // Test with remote metastore service - int port = MetaStoreTestUtils.findFreePort(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java index fa5ae809a810f3bcd79db076af919ffe83522b9e..74fb6e87e53c4c2e5e00cf4c59b4b3fa4066b09d 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java @@ -62,9 +62,6 @@ protected HiveConf createHiveConf() throws Exception { @Before public void setUp() throws Exception { - - int port = MetaStoreTestUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -74,7 +71,8 @@ public void setUp() throws Exception { InjectableDummyAuthenticator.class.getName()); clientHiveConf = createHiveConf(); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), clientHiveConf); + int port = MetaStoreTestUtils + .startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), clientHiveConf); // Turn off client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java index 102af6a08a5eb4336b303eacae0159b1ecd23758..9dc15a4f7ff4da294ee44257ba4ba44c99d47d99 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java @@ -53,8 +53,6 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname, @@ -62,7 +60,7 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); clientHiveConf = new HiveConf(this.getClass()); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java index 60273e7aad22b61c9963cd52186445d93ee7b947..cdde57c75685f516737cd0caf237da5b9fa2d4e9 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java @@ -58,13 +58,11 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - // Turn off metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, ""); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); clientHiveConf = new HiveConf(this.getClass()); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java index f0196c605c65b8d818d312fb7be673e7a869501a..e7cb310af3f8120d086a97dd378e46c7e802807b 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java @@ -84,8 +84,6 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreTestUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -98,8 +96,7 @@ protected void setUp() throws Exception { InjectableDummyAuthenticator.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, ""); - - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); clientHiveConf = createHiveConf(); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java index 62835b090ca52439ef5e737355017554a3ede8d3..5e3095610efb73d86be0a381c407268d9f2d7046 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java @@ -47,10 +47,6 @@ @BeforeClass public static void setUp() throws Exception { - - - int port = MetaStoreTestUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -62,7 +58,7 @@ public static void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge()); clientHiveConf = new HiveConf(); diff --git itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index e554cc83e513b9a40183510fbc1338c52c58c1f0..8b278ee87a752e8db419c64badffe548a65a3a8b 100644 --- itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -75,6 +75,7 @@ private final boolean cleanupLocalDirOnStartup; private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; + private boolean usePortsFromConf = false; public enum MiniClusterType { MR, @@ -227,6 +228,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM this.isMetastoreRemote = isMetastoreRemote; this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; + this.usePortsFromConf = usePortsFromConf; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); FileSystem fs; @@ -334,9 +336,9 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsF public void start(Map confOverlay) throws Exception { if (isMetastoreRemote) { - int metaStorePort = MetaStoreTestUtils.findFreePort(); + int metaStorePort = MetaStoreTestUtils + .startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), getHiveConf()); getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); - MetaStoreTestUtils.startMetaStore(metaStorePort, HadoopThriftAuthBridge.getBridge(), getHiveConf()); } hiveServer2 = new HiveServer2(); @@ -345,7 +347,34 @@ public void start(Map confOverlay) throws Exception { setConfProperty(entry.getKey(), entry.getValue()); } hiveServer2.init(getHiveConf()); - hiveServer2.start(); + + Exception HS2Exception = null; + boolean HS2Started = false; + for (int tryCount = 0; tryCount < MetaStoreTestUtils.retryCount; tryCount++) { + try { + hiveServer2.start(); + tryCount = MetaStoreTestUtils.retryCount; + HS2Started = true; + } catch (Exception t) { + HS2Exception = t; + if (usePortsFromConf) { + tryCount = MetaStoreTestUtils.retryCount; + HS2Started = false; + } else { + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, + MetaStoreTestUtils.findFreePort()); + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, + MetaStoreTestUtils.findFreePort()); + + hiveServer2.init(getHiveConf()); + } + } + } + + if (!HS2Started) { + throw(HS2Exception); + } + waitForStartup(); setStarted(true); } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java index 28353c3780f82bc69af28e7e56da535e257f9871..c8b0ad854604efdc493724c70ee14d8c0db9843d 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; import java.io.IOException; +import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -35,6 +36,7 @@ public class MetaStoreTestUtils { private static final Logger LOG = LoggerFactory.getLogger("hive.log"); + public static final int retryCount = 10; public static int startMetaStore() throws Exception { return MetaStoreTestUtils.startMetaStore(HadoopThriftAuthBridge.getBridge(), null); @@ -76,6 +78,28 @@ public void run() { MetaStoreTestUtils.loopUntilHMSReady(port); } + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge) throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(bridge, null); + } + + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge, HiveConf conf) + throws Exception { + Exception metaStoreException = null; + int metaStorePort = 0; + + for (int tryCount = 0; tryCount < MetaStoreTestUtils.retryCount; tryCount++) { + try { + metaStorePort = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(metaStorePort, HadoopThriftAuthBridge.getBridge(), conf); + return metaStorePort; + } catch (ConnectException ce) { + metaStoreException = ce; + } + } + + throw metaStoreException; + } + /** * A simple connect test to make sure that the metastore is up * @throws Exception