From 0fec534328b65606415fbcb2ea468516e4a9ff21 Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Tue, 5 Dec 2017 14:25:37 -0600 Subject: [PATCH] HBASE-19289 Add flag to disable stream capability enforcement --- .../src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java | 3 +++ .../apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java | 6 ++++-- hbase-procedure/src/test/resources/hbase-site.xml | 5 +++++ .../org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java | 5 +++-- .../org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java | 3 ++- .../test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java | 1 - hbase-server/src/test/resources/hbase-site.xml | 5 +++++ 7 files changed, 22 insertions(+), 6 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java index bdf148eff1..e745bdb2ec 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java @@ -62,6 +62,9 @@ public abstract class CommonFSUtils { /** Parameter name for HBase WAL directory */ public static final String HBASE_WAL_DIR = "hbase.wal.dir"; + /** Parameter to disable stream capability enforcement checks */ + public static final String UNSAFE_STREAM_CAPABILITY_ENFORCE = "hbase.unsafe.stream.capability.enforce"; + /** Full access permissions (starting point for a umask) */ public static final String FULL_RWX_PERMISSIONS = "777"; diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java index f49833c77b..84cda6526f 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java @@ -131,6 +131,7 @@ public class WALProcedureStore extends ProcedureStoreBase { private final FileSystem fs; private final Path walDir; private final Path walArchiveDir; + private final boolean enforceStreamCapability; private final AtomicReference syncException = new AtomicReference<>(); private final AtomicBoolean loading = new AtomicBoolean(true); @@ -205,6 +206,7 @@ public class WALProcedureStore extends ProcedureStoreBase { this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.fs = walDir.getFileSystem(conf); + this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); // Create the log directory for the procedure store if (!fs.exists(walDir)) { @@ -1028,8 +1030,8 @@ public class WALProcedureStore extends ProcedureStoreBase { // ensure that we can provide the level of data safety we're configured // to provide. final String durability = useHsync ? "hsync" : "hflush"; - if (!(CommonFSUtils.hasCapability(newStream, durability))) { - throw new IllegalStateException("The procedure WAL relies on the ability to " + durability + + if (enforceStreamCapability && !(CommonFSUtils.hasCapability(newStream, durability))) { + throw new IllegalStateException("The procedure WAL relies on the ability to " + durability + " for proper operation during component failures, but the underlying filesystem does " + "not support doing so. Please check the config value of '" + USE_HSYNC_CONF_KEY + "' to set the desired level of robustness and ensure the config value of '" + diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml index d3501275a5..f3f955f35c 100644 --- a/hbase-procedure/src/test/resources/hbase-site.xml +++ b/hbase-procedure/src/test/resources/hbase-site.xml @@ -32,4 +32,9 @@ procedure to have an owner + + hbase.unsafe.stream.capability.enforce + false + We don't need to enforce stream capabilities in tests using local FS + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java index 6a7e4fa0b3..583759b66d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java @@ -73,8 +73,9 @@ public final class AsyncFSOutputHelper { // After we create the stream but before we attempt to use it at all // ensure that we can provide the level of data safety we're configured // to provide. - if (!(CommonFSUtils.hasCapability(fsOut, "hflush") && - CommonFSUtils.hasCapability(fsOut, "hsync"))) { + if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) && + !(CommonFSUtils.hasCapability(fsOut, "hflush") && + CommonFSUtils.hasCapability(fsOut, "hsync"))) { throw new CommonFSUtils.StreamLacksCapabilityException("hflush and hsync"); } final ExecutorService flushExecutor = diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java index d1e72f764c..0b52e394df 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java @@ -92,7 +92,8 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter this.output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, null); // TODO Be sure to add a check for hsync if this branch includes HBASE-19024 - if (!(CommonFSUtils.hasCapability(output, "hflush"))) { + if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) && + !(CommonFSUtils.hasCapability(output, "hflush"))) { throw new StreamLacksCapabilityException("hflush"); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index 39ed9dfa5b..fb26e3daf3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -150,7 +150,6 @@ public class TestHStore { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString(); - /** * Setup * @throws IOException diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 64a1964435..e368c82396 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -158,4 +158,9 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.unsafe.stream.capability.enforce + false + We don't need to enforce stream capabilities in tests using local FS + -- 2.15.0