From 64a0da462dcc28a5a87a53f8dfcb0715da6bf20c Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Mon, 29 Jan 2018 14:32:39 -0600 Subject: [PATCH] HBASE-19841 Every HTU should be local until DFS starts --- .../apache/hadoop/hbase/HBaseTestingUtility.java | 104 ++++++++++++--------- .../hadoop/hbase/regionserver/TestHStore.java | 4 +- hbase-server/src/test/resources/hbase-site.xml | 9 -- 3 files changed, 62 insertions(+), 55 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 75abd5e219..9db0f3db8e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -119,6 +119,7 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; @@ -298,10 +299,37 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { public static final Collection BLOOM_AND_COMPRESSION_COMBINATIONS = bloomAndCompressionCombinations(); + + /** + *

Create an HBaseTestingUtility using a default configuration. + * + *

Initially, all tmp files are written to a local test data directory. + * Once {@link #startMiniDFSCluster} is called, either directly or via + * {@link #startMiniCluster()}, tmp data will be written to the DFS directory instead. + * + *

Previously, there was a distinction between the type of utility returned by + * {@link #createLocalHTU()} and this constructor; this is no longer the case. All + * HBaseTestingUtility objects will behave as local until a DFS cluster is started, + * at which point they will switch to using mini DFS for storage. + */ public HBaseTestingUtility() { this(HBaseConfiguration.create()); } + /** + *

Create an HBaseTestingUtility using a given configuration. + * + *

Initially, all tmp files are written to a local test data directory. + * Once {@link #startMiniDFSCluster} is called, either directly or via + * {@link #startMiniCluster()}, tmp data will be written to the DFS directory instead. + * + *

Previously, there was a distinction between the type of utility returned by + * {@link #createLocalHTU()} and this constructor; this is no longer the case. All + * HBaseTestingUtility objects will behave as local until a DFS cluster is started, + * at which point they will switch to using mini DFS for storage. + * + * @param conf The configuration to use for further operations + */ public HBaseTestingUtility(Configuration conf) { super(conf); @@ -321,34 +349,32 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { LOG.debug("Config property " + HConstants.REGIONSERVER_PORT + " changed to -1"); } } + + // Every cluster is a local cluster until we start DFS + String dataTestDir = getDataTestDir().toString(); + conf.set("fs.defaultFS","file:///"); + conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); + LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir); + conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,false); + localMode = true; } /** - * Create an HBaseTestingUtility where all tmp files are written to the local test data dir. - * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper - * test dir. Use this when you aren't using an Mini HDFS cluster. - * @return HBaseTestingUtility that use local fs for temp files. + * @deprecated use {@link HBaseTestingUtility#HBaseTestingUtility()} instead + * @return a normal HBaseTestingUtility */ + @Deprecated public static HBaseTestingUtility createLocalHTU() { - Configuration c = HBaseConfiguration.create(); - return createLocalHTU(c); + return new HBaseTestingUtility(); } /** - * Create an HBaseTestingUtility where all tmp files are written to the local test data dir. - * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper - * test dir. Use this when you aren't using an Mini HDFS cluster. - * @param c Configuration (will be modified) - * @return HBaseTestingUtility that use local fs for temp files. + * @deprecated use {@link HBaseTestingUtility#HBaseTestingUtility(Configuration)} instead + * @return a normal HBaseTestingUtility */ + @Deprecated public static HBaseTestingUtility createLocalHTU(Configuration c) { - HBaseTestingUtility htu = new HBaseTestingUtility(c); - String dataTestDir = htu.getDataTestDir().toString(); - htu.getConfiguration().set("fs.defaultFS","file:///"); - htu.getConfiguration().set(HConstants.HBASE_DIR, "file://" + dataTestDir); - LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir); - htu.localMode = true; - return htu; + return new HBaseTestingUtility(c); } /** @@ -612,30 +638,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { */ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[]) throws Exception { - createDirsAndSetProperties(); - EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); - - // Error level to skip some warnings specific to the minicluster. See HBASE-4709 - org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class). - setLevel(org.apache.log4j.Level.ERROR); - org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class). - setLevel(org.apache.log4j.Level.ERROR); - - TraceUtil.initTracer(conf); - - this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true, - true, null, null, hosts, null); - - // Set this just-started cluster as our filesystem. - setFs(); - - // Wait for the cluster to be totally up - this.dfsCluster.waitClusterUp(); - - //reset the test directory for test file system - dataTestDirOnTestFS = null; - - return this.dfsCluster; + return startMiniDFSCluster(servers, null, hosts); } private void setFs() throws IOException { @@ -643,19 +646,32 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { LOG.info("Skipping setting fs because dfsCluster is null"); return; } + localMode = false; FileSystem fs = this.dfsCluster.getFileSystem(); FSUtils.setFsDefault(this.conf, new Path(fs.getUri())); + // re-enable this check with dfs + conf.unset(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE); } public MiniDFSCluster startMiniDFSCluster(int servers, final String racks[], String hosts[]) throws Exception { + // unset using the local file system + conf.unset(HConstants.HBASE_DIR); + createDirsAndSetProperties(); + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + + // Error level to skip some warnings specific to the minicluster. See HBASE-4709 + org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class). + setLevel(org.apache.log4j.Level.ERROR); + org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class). + setLevel(org.apache.log4j.Level.ERROR); + this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true, true, null, racks, hosts, null); // Set this just-started cluster as our filesystem. - FileSystem fs = this.dfsCluster.getFileSystem(); - FSUtils.setFsDefault(this.conf, new Path(fs.getUri())); + setFs(); // Wait for the cluster to be totally up this.dfsCluster.waitClusterUp(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index 9d6aedb4ee..8988d398e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -241,7 +241,7 @@ public class TestHStore { public void testFlushSizeSizing() throws Exception { LOG.info("Setting up a faulty file system that cannot write in " + this.name.getMethodName()); - final Configuration conf = HBaseConfiguration.create(); + final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); // Only retry once. conf.setInt("hbase.hstore.flush.retries.number", 1); User user = User.createUserForTesting(conf, this.name.getMethodName(), @@ -668,7 +668,7 @@ public class TestHStore { public void testHandleErrorsInFlush() throws Exception { LOG.info("Setting up a faulty file system that cannot write"); - final Configuration conf = HBaseConfiguration.create(); + final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); User user = User.createUserForTesting(conf, "testhandleerrorsinflush", new String[]{"foo"}); // Inject our faulty LocalFileSystem diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index dbdf7765d7..64a1964435 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -158,13 +158,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.unsafe.stream.capability.enforce - false - - Controls whether HBase will check for stream capabilities (hflush/hsync). - Disable this if you intend to run on LocalFileSystem. - WARNING: Doing so may expose you to additional risk of data loss! - - -- 2.15.1