commit af0f8e7ea2596124aedb6a374341fe125fb6083f Author: Janaki Lahorani Date: Sat Nov 25 10:42:28 2017 -0800 HIVE-18147: Retry getting a new port number for 10 times if the starting of HMS or HS2 fails with the picked port Change-Id: Ic0c9376d5f684999d869080a1c824d020220949b diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 66a5dd43d544c29ece588e54673de581b7625d79..2375f5d4d1e6381db8f054df847db7ed18121528 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -79,9 +79,7 @@ protected void setUp() throws Exception { return; } - - msPort = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(msPort, ShimLoader.getHadoopThriftAuthBridge()); + msPort = MetaStoreUtils.startMetaStoreWithRetry(); isServerRunning = true; diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index 69874bcae88a2a4c0e5215e810ffb9bd9c0a7bb4..777fc9c4588bebb087c7b8a0258e045b77009f64 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -104,10 +104,8 @@ public static void setup() throws Exception { return; } - msPort = MetaStoreUtils.findFreePort(); + msPort = MetaStoreUtils.startMetaStoreWithRetry(); - MetaStoreUtils.startMetaStore(msPort, ShimLoader - .getHadoopThriftAuthBridge()); Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); diff --git hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java index b62716ec80da5b68426a73ed9b03864eb76056b2..841e2ffa0edc4d7761c7cf5cedd155392c94bdd8 100644 --- hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java +++ hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java @@ -70,21 +70,36 @@ private static final String charSet = "UTF-8"; @BeforeClass - public static void startHebHcatInMem() { - int webhcatPort = 50111; - try { - //in case concurrent tests are running on the same machine - webhcatPort = MetaStoreUtils.findFreePort(); + public static void startHebHcatInMem() throws Exception { + Exception webhcatException = null; + int webhcatPort = 0; + boolean webhcatStarted = false; + + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + if (tryCount == MetaStoreUtils.RETRY_COUNT - 1) { + // down to the last try. try default port 50111 + webhcatPort = 50111; + } else { + webhcatPort = MetaStoreUtils.findFreePort(); + } + templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); + templetonServer = new Main(new String[] { "-D" + AppConfig.UNIT_TEST_MODE + "=true", + "-D" + AppConfig.PORT + "=" + webhcatPort }); + LOG.info("Starting Main; WebHCat using port: " + webhcatPort); + templetonServer.run(); + LOG.info("Main started"); + webhcatStarted = true; + break; + } catch (Exception ce) { + LOG.info("Attempt to Start WebHCat using port: " + webhcatPort + " failed"); + webhcatException = ce; + } } - catch (IOException ex) { - LOG.warn("Unable to find free port; using default: " + webhcatPort); + + if (!webhcatStarted) { + throw webhcatException; } - templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); - templetonServer = new Main(new String[] {"-D" + - AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort}); - LOG.info("Starting Main; WebHCat using port: " + webhcatPort); - templetonServer.run(); - LOG.info("Main started"); } @AfterClass diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java index 341c7ba533ad7b7a60af10d5815e69e68ec24d64..e72a5850884d9d2d5868c33c410376986e4a36e7 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java @@ -57,8 +57,7 @@ protected static void setup() throws Exception { hiveConf = new HiveConf(); if (isRemoteMetastoreMode) { - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); } hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 1073abb4e1b8023b1237887e9cd99f0f0024526d..42859d48da66f37d26a84bef5e320b0e6ac84457 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -173,9 +173,8 @@ public static void setUp() throws Exception { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName()); UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName()); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); SessionState.start(new CliSessionState(hiveConf)); msc = new HiveMetaStoreClient(hiveConf); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java index 600aef4c2bd0395627d8e787e3254e18b1b962ac..4aa527cc822ed0517b5d48f386a299326bd34555 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreStatsMerge.java @@ -76,8 +76,7 @@ protected void setUp() throws Exception { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java index d6e4fb7632e3cba1daf5910666dc3f5a545cac67..e999bfbdcd18628802d88c9ea03be0631d72f2dd 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java @@ -73,8 +73,7 @@ protected void setUp() throws Exception { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java index 1e78ff1b6ff84c2f2b3cf9623ca78fb843b54eb5..1158b766792e909e90ce2a59834ed9b590675738 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java @@ -47,8 +47,7 @@ protected void setUp() throws Exception { DummyPreListener.class.getName()); System.setProperty("hive.metastore.end.function.listeners", DummyEndFunctionListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index fd4527e65367a15ef2d07200ec3e8ceecfa541ee..2ad58cf74021e19044739072c3bc6aa165d01516 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -92,8 +92,7 @@ protected void setUp() throws Exception { System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java index 0c3e703761b3244eab3d0dc68349cb3acc5ca6a5..84ebbe15a7aeaca14eff89d2a7bc78d5fef73a94 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java @@ -50,8 +50,7 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname, DummyRawStoreControlledCommit.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java index e8171e56a668dea7de005e0694b2d30bb4d2cf73..b6426ffe442cb1b3fda3ebaa8d05549b0dbf261c 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java @@ -42,8 +42,7 @@ protected void setUp() throws Exception { super.setUp(); System.setProperty("hive.metastore.init.hooks", DummyMetaStoreInitListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index b462b2a893268b086bbc3ca38f5980adbe6420bc..8d6a45b8777db8ce9b7702d30b700dffbc23acf9 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -42,10 +42,7 @@ @BeforeClass public static void before() throws Exception { - int port = MetaStoreUtils.findFreePort(); - hiveConf = new HiveConf(TestMetaStoreMetrics.class); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); @@ -58,7 +55,9 @@ public static void before() throws Exception { metrics = (CodahaleMetrics) MetricsFactory.getInstance(); //Increments one HMS connection - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); + int port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf); + + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); //Increments one HMS connection (Hive.get()) SessionState.start(new CliSessionState(hiveConf)); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index ef02968e22363d537f58b6054266bf9bc87033ae..225cf0aa112d07f34d9c7cd3e8d18dc1470f5306 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -42,9 +42,8 @@ protected void setUp() throws Exception { return; } - port = MetaStoreUtils.findFreePort(); + port = MetaStoreUtils.startMetaStoreWithRetry(hiveConf); System.out.println("Starting MetaStore Server on port " + port); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); isServerStarted = true; // This is default case with setugi off for both client and server diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java index 63eea2710a2ac5571abf467226e873165f002b10..1c1c3e62f95f49104aeae007f0a7418c3ddd2a86 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -48,11 +48,10 @@ protected void setUp() throws Exception { return; } - int port = MetaStoreUtils.findFreePort(); - System.out.println("Starting MetaStore Server on port " + port); System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname, IpAddressListener.class.getName()); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); + System.out.println("Starting MetaStore Server on port " + port); isServerStarted = true; // This is default case with setugi off for both client and server diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java index 99c09b7823962c213d0919950d05b562d870d812..abe241615282ed4a1312456f686370f986f51423 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -53,8 +53,7 @@ protected void setUp() throws Exception { super.setUp(); System.setProperty("hive.metastore.pre.event.listeners", AlternateFailurePreListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index bfb25aa800463abfa252b451731431f43a3318b2..a9bb4ad45e8a5f1aac3fc5885ca066a7e249a50a 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -79,8 +79,7 @@ protected void setUp() throws Exception { SessionState.start(conf); // Test with remote metastore service - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java index bf6b7e1c41b6ec579b305ee9e23bbc3349b3b631..63cd732b5a3580272f855034a1b454623724b16d 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java @@ -100,8 +100,7 @@ public static void baseSetup() throws Exception { conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true); conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + MetaStoreUtils.startMetaStoreWithRetry(); SessionState.start(new CliSessionState(conf)); driver = new Driver(conf); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java index 36f31f4d9ceb668c42e352eb95b05777dd361ba4..f6ac49342a919ef2b39f17957d5243899ee5d7dc 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java @@ -63,9 +63,6 @@ protected HiveConf createHiveConf() throws Exception { @Before public void setUp() throws Exception { - - int port = MetaStoreUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -75,7 +72,7 @@ public void setUp() throws Exception { InjectableDummyAuthenticator.class.getName()); clientHiveConf = createHiveConf(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), clientHiveConf); + int port = MetaStoreUtils.startMetaStoreWithRetry(clientHiveConf); // Turn off client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java index b0da8845fc61a068f9db5c42d453ea49e17179f5..b357a554b2ce8051192573578b9d793aec1d7197 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java @@ -53,8 +53,6 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname, @@ -62,7 +60,7 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java index d895da89fcd3ac096f1923d0e341fad3ee413205..312d4aa2b82932d2f476f533a01f24fd41e382cf 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java @@ -58,13 +58,11 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - // Turn off metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, ""); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(this.getClass()); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java index 19dc9cf85b847d9fa9d23adf9834868f5e85e345..e11d28c2eeadf18fc40f923dde51061bb73a8d0f 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java @@ -84,8 +84,6 @@ protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -98,8 +96,7 @@ protected void setUp() throws Exception { InjectableDummyAuthenticator.class.getName()); System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, ""); - - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = createHiveConf(); diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java index 5c9bf05373b54fb7acd184ddf5d7642478340e7b..45b443f030b6b885e0eebace7445ab9352450820 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java @@ -47,10 +47,6 @@ @BeforeClass public static void setUp() throws Exception { - - - int port = MetaStoreUtils.findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -62,7 +58,7 @@ public static void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + int port = MetaStoreUtils.startMetaStoreWithRetry(); clientHiveConf = new HiveConf(); diff --git itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java index 71483073decbbd862ac4be3ceaae12588da56479..a73ea9fc577e4d2247c8e656ab7fc0b31a7b31f9 100644 --- itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java +++ itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java @@ -68,13 +68,27 @@ public static void tearDownAfterClass() throws Exception { protected static void startHiveServer2WithConf(HiveServer2 hiveServer2, HiveConf hiveConf) throws Exception { - hiveServer2.init(hiveConf); // Start HiveServer2 with given config // Fail if server doesn't start - try { - hiveServer2.start(); - } catch (Throwable t) { - t.printStackTrace(); + + Throwable hs2Exception = null; + boolean hs2Started = false; + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2.init(hiveConf); + hiveServer2.start(); + hs2Started = true; + break; + } catch (Throwable t) { + hs2Exception = t; + port = MetaStoreUtils.findFreePort(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveServer2 = new HiveServer2(); + } + } + + if (!hs2Started) { + hs2Exception.printStackTrace(); fail(); } // Wait for startup to complete diff --git itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index ebc4c10b6408bc8693090a7fd76f6f08d7803918..2bf2461cfa0a683a9988409e0ff7abfe09f2c355 100644 --- itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -74,6 +74,7 @@ private final boolean cleanupLocalDirOnStartup; private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; + private boolean usePortsFromConf = false; public enum MiniClusterType { MR, @@ -226,6 +227,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM this.isMetastoreRemote = isMetastoreRemote; this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; + this.usePortsFromConf = usePortsFromConf; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); FileSystem fs; @@ -333,19 +335,42 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsF public void start(Map confOverlay) throws Exception { if (isMetastoreRemote) { - int metaStorePort = MetaStoreUtils.findFreePort(); + int metaStorePort = MetaStoreUtils.startMetaStoreWithRetry(getHiveConf()); getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); - MetaStoreUtils.startMetaStore(metaStorePort, - ShimLoader.getHadoopThriftAuthBridge(), getHiveConf()); } - hiveServer2 = new HiveServer2(); // Set confOverlay parameters for (Map.Entry entry : confOverlay.entrySet()) { setConfProperty(entry.getKey(), entry.getValue()); } - hiveServer2.init(getHiveConf()); - hiveServer2.start(); + + Exception hs2Exception = null; + boolean hs2Started = false; + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2 = new HiveServer2(); + hiveServer2.init(getHiveConf()); + hiveServer2.start(); + hs2Started = true; + break; + } catch (Exception t) { + hs2Exception = t; + if (usePortsFromConf) { + hs2Started = false; + break; + } else { + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, + MetaStoreUtils.findFreePort()); + HiveConf.setIntVar(getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, + MetaStoreUtils.findFreePort()); + } + } + } + + if (!hs2Started) { + throw(hs2Exception); + } + waitForStartup(); setStarted(true); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 3eb8c36337ac4d342681649384925729ac84b2a4..13bdf8e08384df472c7465a3247fc1f4d2d3d4b7 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; @@ -101,6 +102,8 @@ public static final String DATABASE_WAREHOUSE_SUFFIX = ".db"; + public static final int RETRY_COUNT = 10; + // Right now we only support one special character '/'. // More special characters can be added accordingly in the future. // NOTE: @@ -1250,6 +1253,36 @@ public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf c return port; } + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge) throws Exception { + return startMetaStoreWithRetry(bridge, null); + } + + public static int startMetaStoreWithRetry(HiveConf conf) throws Exception { + return startMetaStoreWithRetry(ShimLoader.getHadoopThriftAuthBridge(), conf); + } + + public static int startMetaStoreWithRetry() throws Exception { + return startMetaStoreWithRetry(ShimLoader.getHadoopThriftAuthBridge(), null); + } + + public static int startMetaStoreWithRetry(final HadoopThriftAuthBridge bridge, HiveConf conf) + throws Exception { + Exception metaStoreException = null; + int metaStorePort = 0; + + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + metaStorePort = findFreePort(); + startMetaStore(metaStorePort, bridge, conf); + return metaStorePort; + } catch (ConnectException ce) { + metaStoreException = ce; + } + } + + throw metaStoreException; + } + public static int startMetaStore(HiveConf conf) throws Exception { return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf); } diff --git service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java index 6fec947d629bcecc7a20c5e764243bb837012880..8977a96414337475b97394a559363e4778c1a59c 100644 --- service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java +++ service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java @@ -95,15 +95,30 @@ public static void tearDownAfterClass() throws Exception { } protected static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception { - hiveServer2.init(hiveConf); // Start HiveServer2 with given config // Fail if server doesn't start - try { - hiveServer2.start(); - } catch (Throwable t) { - t.printStackTrace(); + + Throwable hs2Exception = null; + boolean hs2Started = false; + for (int tryCount = 0; tryCount < MetaStoreUtils.RETRY_COUNT; tryCount++) { + try { + hiveServer2.init(hiveConf); + hiveServer2.start(); + hs2Started = true; + break; + } catch (Throwable t) { + hs2Exception = t; + port = MetaStoreUtils.findFreePort(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveServer2 = new HiveServer2(); + } + } + + if (!hs2Started) { + hs2Exception.printStackTrace(); fail(); } + // Wait for startup to complete Thread.sleep(2000); System.out.println("HiveServer2 started on port " + port);