From 2afaaa8c0ed24c944ecd8f7a19a8e28ef28d64bd Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Mon, 29 Jan 2018 14:32:39 -0600 Subject: [PATCH] HBASE-19841 Every HTU should be local until DFS starts --- .../apache/hadoop/hbase/util/CommonFSUtils.java | 8 +- .../org/apache/hadoop/hbase/HBaseTestCase.java | 2 +- .../apache/hadoop/hbase/HBaseTestingUtility.java | 105 ++++++++++++--------- .../coprocessor/TestRegionObserverStacking.java | 2 +- .../hbase/io/asyncfs/TestLocalAsyncOutput.java | 4 +- .../hadoop/hbase/regionserver/TestHStore.java | 4 +- hbase-server/src/test/resources/hbase-site.xml | 9 -- 7 files changed, 74 insertions(+), 60 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java index 9efec07915..5b46de9733 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java @@ -394,7 +394,13 @@ public abstract class CommonFSUtils { public static FileSystem getWALFileSystem(final Configuration c) throws IOException { Path p = getWALRootDir(c); - return p.getFileSystem(c); + FileSystem fs = p.getFileSystem(c); + // hadoop-core does fs caching, so need to propogate this if set + String enforceStreamCapability = c.get(UNSAFE_STREAM_CAPABILITY_ENFORCE); + if (enforceStreamCapability != null) { + fs.getConf().set(UNSAFE_STREAM_CAPABILITY_ENFORCE, enforceStreamCapability); + } + return fs; } private static boolean isValidWALRootDir(Path walDir, final Configuration c) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java index 5f2ffb2f33..96f3d55608 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java @@ -72,7 +72,7 @@ public abstract class HBaseTestCase extends TestCase { protected final HBaseTestingUtility testUtil = new HBaseTestingUtility(); - public volatile Configuration conf = HBaseConfiguration.create(); + public volatile Configuration conf = testUtil.getConfiguration(); public final FSTableDescriptors fsTableDescriptors; { try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 75abd5e219..93fd736f20 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -119,6 +119,7 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; @@ -298,10 +299,37 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { public static final Collection BLOOM_AND_COMPRESSION_COMBINATIONS = bloomAndCompressionCombinations(); + + /** + *

Create an HBaseTestingUtility using a default configuration. + * + *

Initially, all tmp files are written to a local test data directory. + * Once {@link #startMiniDFSCluster} is called, either directly or via + * {@link #startMiniCluster()}, tmp data will be written to the DFS directory instead. + * + *

Previously, there was a distinction between the type of utility returned by + * {@link #createLocalHTU()} and this constructor; this is no longer the case. All + * HBaseTestingUtility objects will behave as local until a DFS cluster is started, + * at which point they will switch to using mini DFS for storage. + */ public HBaseTestingUtility() { this(HBaseConfiguration.create()); } + /** + *

Create an HBaseTestingUtility using a given configuration. + * + *

Initially, all tmp files are written to a local test data directory. + * Once {@link #startMiniDFSCluster} is called, either directly or via + * {@link #startMiniCluster()}, tmp data will be written to the DFS directory instead. + * + *

Previously, there was a distinction between the type of utility returned by + * {@link #createLocalHTU()} and this constructor; this is no longer the case. All + * HBaseTestingUtility objects will behave as local until a DFS cluster is started, + * at which point they will switch to using mini DFS for storage. + * + * @param conf The configuration to use for further operations + */ public HBaseTestingUtility(Configuration conf) { super(conf); @@ -321,34 +349,32 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { LOG.debug("Config property " + HConstants.REGIONSERVER_PORT + " changed to -1"); } } + + // Every cluster is a local cluster until we start DFS + String dataTestDir = getDataTestDir().toString(); + conf.set("fs.defaultFS","file:///"); + conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); + LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); + conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,false); + localMode = true; } /** - * Create an HBaseTestingUtility where all tmp files are written to the local test data dir. - * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper - * test dir. Use this when you aren't using an Mini HDFS cluster. - * @return HBaseTestingUtility that use local fs for temp files. + * @deprecated use {@link HBaseTestingUtility#HBaseTestingUtility()} instead + * @return a normal HBaseTestingUtility */ + @Deprecated public static HBaseTestingUtility createLocalHTU() { - Configuration c = HBaseConfiguration.create(); - return createLocalHTU(c); + return new HBaseTestingUtility(); } /** - * Create an HBaseTestingUtility where all tmp files are written to the local test data dir. - * It is needed to properly base FSUtil.getRootDirs so that they drop temp files in the proper - * test dir. Use this when you aren't using an Mini HDFS cluster. - * @param c Configuration (will be modified) - * @return HBaseTestingUtility that use local fs for temp files. + * @deprecated use {@link HBaseTestingUtility#HBaseTestingUtility(Configuration)} instead + * @return a normal HBaseTestingUtility */ + @Deprecated public static HBaseTestingUtility createLocalHTU(Configuration c) { - HBaseTestingUtility htu = new HBaseTestingUtility(c); - String dataTestDir = htu.getDataTestDir().toString(); - htu.getConfiguration().set("fs.defaultFS","file:///"); - htu.getConfiguration().set(HConstants.HBASE_DIR, "file://" + dataTestDir); - LOG.debug("Setting " + HConstants.HBASE_DIR + " to " + dataTestDir); - htu.localMode = true; - return htu; + return new HBaseTestingUtility(c); } /** @@ -612,30 +638,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { */ public MiniDFSCluster startMiniDFSCluster(int servers, final String hosts[]) throws Exception { - createDirsAndSetProperties(); - EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); - - // Error level to skip some warnings specific to the minicluster. See HBASE-4709 - org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class). - setLevel(org.apache.log4j.Level.ERROR); - org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class). - setLevel(org.apache.log4j.Level.ERROR); - - TraceUtil.initTracer(conf); - - this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true, - true, null, null, hosts, null); - - // Set this just-started cluster as our filesystem. - setFs(); - - // Wait for the cluster to be totally up - this.dfsCluster.waitClusterUp(); - - //reset the test directory for test file system - dataTestDirOnTestFS = null; - - return this.dfsCluster; + return startMiniDFSCluster(servers, null, hosts); } private void setFs() throws IOException { @@ -643,25 +646,39 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { LOG.info("Skipping setting fs because dfsCluster is null"); return; } + localMode = false; FileSystem fs = this.dfsCluster.getFileSystem(); FSUtils.setFsDefault(this.conf, new Path(fs.getUri())); + + // re-enable this check with dfs + conf.unset(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE); } public MiniDFSCluster startMiniDFSCluster(int servers, final String racks[], String hosts[]) throws Exception { createDirsAndSetProperties(); + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + + // Error level to skip some warnings specific to the minicluster. See HBASE-4709 + org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.util.MBeans.class). + setLevel(org.apache.log4j.Level.ERROR); + org.apache.log4j.Logger.getLogger(org.apache.hadoop.metrics2.impl.MetricsSystemImpl.class). + setLevel(org.apache.log4j.Level.ERROR); + this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true, true, null, racks, hosts, null); // Set this just-started cluster as our filesystem. - FileSystem fs = this.dfsCluster.getFileSystem(); - FSUtils.setFsDefault(this.conf, new Path(fs.getUri())); + setFs(); // Wait for the cluster to be totally up this.dfsCluster.waitClusterUp(); //reset the test directory for test file system dataTestDirOnTestFS = null; + String dataTestDir = getDataTestDir().toString(); + conf.set(HConstants.HBASE_DIR, dataTestDir); + LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); return this.dfsCluster; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java index cbc00d3f03..5546bfa10d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverStacking.java @@ -146,7 +146,7 @@ public class TestRegionObserverStacking extends TestCase { byte[] A = Bytes.toBytes("A"); byte[][] FAMILIES = new byte[][] { A } ; - Configuration conf = HBaseConfiguration.create(); + Configuration conf = TEST_UTIL.getConfiguration(); HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES); RegionCoprocessorHost h = region.getCoprocessorHost(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java index f2f91425a7..406af17b52 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestLocalAsyncOutput.java @@ -22,7 +22,7 @@ import java.util.concurrent.ExecutionException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.FSUtils; @@ -47,7 +47,7 @@ public class TestLocalAsyncOutput { private static Class CHANNEL_CLASS = NioSocketChannel.class; - private static final HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility(); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @AfterClass public static void tearDownAfterClass() throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index 9d6aedb4ee..8988d398e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -241,7 +241,7 @@ public class TestHStore { public void testFlushSizeSizing() throws Exception { LOG.info("Setting up a faulty file system that cannot write in " + this.name.getMethodName()); - final Configuration conf = HBaseConfiguration.create(); + final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); // Only retry once. conf.setInt("hbase.hstore.flush.retries.number", 1); User user = User.createUserForTesting(conf, this.name.getMethodName(), @@ -668,7 +668,7 @@ public class TestHStore { public void testHandleErrorsInFlush() throws Exception { LOG.info("Setting up a faulty file system that cannot write"); - final Configuration conf = HBaseConfiguration.create(); + final Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); User user = User.createUserForTesting(conf, "testhandleerrorsinflush", new String[]{"foo"}); // Inject our faulty LocalFileSystem diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index dbdf7765d7..64a1964435 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -158,13 +158,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.unsafe.stream.capability.enforce - false - - Controls whether HBase will check for stream capabilities (hflush/hsync). - Disable this if you intend to run on LocalFileSystem. - WARNING: Doing so may expose you to additional risk of data loss! - - -- 2.15.1