From bfd25b608d4195a193393fc5fb192addb3da37c6 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Tue, 8 May 2018 06:53:22 -0700 Subject: [PATCH] HBASE-20544 Make HBTU default to random ports. --- .../src/test/resources/hbase-site.xml | 7 --- .../src/test/resources/hbase-site.xml | 7 --- hbase-backup/src/test/resources/hbase-site.xml | 7 --- hbase-client/src/test/resources/hbase-site.xml | 7 --- hbase-endpoint/src/test/resources/hbase-site.xml | 7 --- hbase-examples/src/test/resources/hbase-site.xml | 7 --- hbase-mapreduce/src/test/resources/hbase-site.xml | 7 --- hbase-procedure/src/test/resources/hbase-site.xml | 7 --- hbase-rest/src/test/resources/hbase-site.xml | 7 --- .../hadoop/hbase/rsgroup/TestEnableRSGroup.java | 4 +- hbase-rsgroup/src/test/resources/hbase-site.xml | 7 --- .../apache/hadoop/hbase/HBaseTestingUtility.java | 16 +----- hbase-server/src/test/resources/hbase-site.xml | 7 --- hbase-shell/src/test/resources/hbase-site.xml | 7 --- hbase-spark-it/src/test/resources/hbase-site.xml | 7 --- hbase-spark/src/test/resources/hbase-site.xml | 7 --- .../hadoop/hbase/TestHBaseTestingUtilSpinup.java | 67 ++++++++++++++++++++++ hbase-thrift/src/test/resources/hbase-site.xml | 7 --- 18 files changed, 71 insertions(+), 121 deletions(-) create mode 100644 hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java diff --git a/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml b/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml +++ b/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml b/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml +++ b/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-backup/src/test/resources/hbase-site.xml b/hbase-backup/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-backup/src/test/resources/hbase-site.xml +++ b/hbase-backup/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-client/src/test/resources/hbase-site.xml +++ b/hbase-client/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-endpoint/src/test/resources/hbase-site.xml b/hbase-endpoint/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-endpoint/src/test/resources/hbase-site.xml +++ b/hbase-endpoint/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-examples/src/test/resources/hbase-site.xml b/hbase-examples/src/test/resources/hbase-site.xml index 697567e9f3..ab4d1cd0b7 100644 --- a/hbase-examples/src/test/resources/hbase-site.xml +++ b/hbase-examples/src/test/resources/hbase-site.xml @@ -25,11 +25,4 @@ hbase.defaults.for.version.skip true - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-mapreduce/src/test/resources/hbase-site.xml b/hbase-mapreduce/src/test/resources/hbase-site.xml index 34802d0e9c..64a1964435 100644 --- a/hbase-mapreduce/src/test/resources/hbase-site.xml +++ b/hbase-mapreduce/src/test/resources/hbase-site.xml @@ -158,11 +158,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml index a1cc27ebf1..114ee8a23c 100644 --- a/hbase-procedure/src/test/resources/hbase-site.xml +++ b/hbase-procedure/src/test/resources/hbase-site.xml @@ -41,11 +41,4 @@ WARNING: Doing so may expose you to additional risk of data loss! - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-rest/src/test/resources/hbase-site.xml b/hbase-rest/src/test/resources/hbase-site.xml index be7b49243a..2bd3ee45a4 100644 --- a/hbase-rest/src/test/resources/hbase-site.xml +++ b/hbase-rest/src/test/resources/hbase-site.xml @@ -139,11 +139,4 @@ Skip sanity checks in tests - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java index 1407a5b94f..04d9d08a8d 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java @@ -49,10 +49,10 @@ public class TestEnableRSGroup { protected static final Logger LOG = LoggerFactory.getLogger(TestEnableRSGroup.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static Configuration conf = TEST_UTIL.getConfiguration(); @BeforeClass public static void setUp() throws Exception { + final Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean(CoprocessorHost.COPROCESSORS_ENABLED_CONF_KEY, true); TEST_UTIL.startMiniCluster(5); } @@ -69,9 +69,9 @@ public class TestEnableRSGroup { TEST_UTIL.getMiniHBaseCluster().waitOnMaster(0); LOG.info("stopped master..."); + final Configuration conf = TEST_UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName()); conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); - TEST_UTIL.getMiniHBaseCluster().setConf(conf); TEST_UTIL.getMiniHBaseCluster().startMaster(); TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(60000); diff --git a/hbase-rsgroup/src/test/resources/hbase-site.xml b/hbase-rsgroup/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-rsgroup/src/test/resources/hbase-site.xml +++ b/hbase-rsgroup/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 8faeb5d293..4bbe811ebf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -61,6 +61,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ClusterMetrics.Option; +import org.apache.hadoop.hbase.LocalHBaseCluster; import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.Admin; @@ -333,20 +334,6 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { // a hbase checksum verification failure will cause unit tests to fail ChecksumUtil.generateExceptionForChecksumFailureForTest(true); - // if conf is provided, prevent contention for ports if other hbase thread(s) are running - if (conf != null) { - if (conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT) - == HConstants.DEFAULT_MASTER_INFOPORT) { - conf.setInt(HConstants.MASTER_INFO_PORT, -1); - LOG.debug("Config property {} changed to -1", HConstants.MASTER_INFO_PORT); - } - if (conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT) - == HConstants.DEFAULT_REGIONSERVER_PORT) { - conf.setInt(HConstants.REGIONSERVER_PORT, -1); - LOG.debug("Config property {} changed to -1", HConstants.REGIONSERVER_PORT); - } - } - // Save this for when setting default file:// breaks things this.conf.set("original.defaultFS", this.conf.get("fs.defaultFS")); @@ -357,6 +344,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,false); + this.conf.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true); } /** diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 34802d0e9c..64a1964435 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -158,11 +158,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-shell/src/test/resources/hbase-site.xml b/hbase-shell/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-shell/src/test/resources/hbase-site.xml +++ b/hbase-shell/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-spark-it/src/test/resources/hbase-site.xml b/hbase-spark-it/src/test/resources/hbase-site.xml index 858d4285cf..99d2ab8d1f 100644 --- a/hbase-spark-it/src/test/resources/hbase-site.xml +++ b/hbase-spark-it/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-spark/src/test/resources/hbase-site.xml b/hbase-spark/src/test/resources/hbase-site.xml index b354a2b386..b3fb0d90c5 100644 --- a/hbase-spark/src/test/resources/hbase-site.xml +++ b/hbase-spark/src/test/resources/hbase-site.xml @@ -154,11 +154,4 @@ Enable replay sanity checks on procedure tests. - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java new file mode 100644 index 0000000000..4ea4fc54cf --- /dev/null +++ b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertFalse; + +import java.util.List; + +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Make sure we can spin up a HBTU without a hbase-site.xml + */ +@Category({MiscTests.class, MediumTests.class}) +public class TestHBaseTestingUtilSpinup { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestHBaseTestingUtilSpinup.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestHBaseTestingUtilSpinup.class); + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + if (!UTIL.getHBaseCluster().waitForActiveAndReadyMaster(30000)) { + throw new RuntimeException("Active master not ready"); + } + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testGetMetaTableRows() throws Exception { + List results = UTIL.getMetaTableRows(); + assertFalse("results should have some entries and is empty.", results.isEmpty()); + } + +} diff --git a/hbase-thrift/src/test/resources/hbase-site.xml b/hbase-thrift/src/test/resources/hbase-site.xml index b354a2b386..b3fb0d90c5 100644 --- a/hbase-thrift/src/test/resources/hbase-site.xml +++ b/hbase-thrift/src/test/resources/hbase-site.xml @@ -154,11 +154,4 @@ Enable replay sanity checks on procedure tests. - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - -- 2.16.1