From 43021d222ffb8a18dd1d605348c67a77f15c08e1 Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Tue, 5 Dec 2017 14:25:37 -0600 Subject: [PATCH] HBASE-19289 Add flag to disable stream capability enforcement --- .../main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java | 3 +++ .../hadoop/hbase/procedure2/store/wal/WALProcedureStore.java | 6 ++++-- hbase-procedure/src/test/resources/hbase-site.xml | 9 +++++++++ .../org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java | 5 +++-- .../apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java | 3 ++- hbase-server/src/test/resources/hbase-site.xml | 9 +++++++++ 6 files changed, 30 insertions(+), 5 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java index 38ae476627..2a83a5d6a1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CommonFSUtils.java @@ -62,6 +62,9 @@ public abstract class CommonFSUtils { /** Parameter name for HBase WAL directory */ public static final String HBASE_WAL_DIR = "hbase.wal.dir"; + /** Parameter to disable stream capability enforcement checks */ + public static final String UNSAFE_STREAM_CAPABILITY_ENFORCE = "hbase.unsafe.stream.capability.enforce"; + /** Full access permissions (starting point for a umask) */ public static final String FULL_RWX_PERMISSIONS = "777"; diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java index f49833c77b..84cda6526f 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java @@ -131,6 +131,7 @@ public class WALProcedureStore extends ProcedureStoreBase { private final FileSystem fs; private final Path walDir; private final Path walArchiveDir; + private final boolean enforceStreamCapability; private final AtomicReference syncException = new AtomicReference<>(); private final AtomicBoolean loading = new AtomicBoolean(true); @@ -205,6 +206,7 @@ public class WALProcedureStore extends ProcedureStoreBase { this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.fs = walDir.getFileSystem(conf); + this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); // Create the log directory for the procedure store if (!fs.exists(walDir)) { @@ -1028,8 +1030,8 @@ public class WALProcedureStore extends ProcedureStoreBase { // ensure that we can provide the level of data safety we're configured // to provide. final String durability = useHsync ? "hsync" : "hflush"; - if (!(CommonFSUtils.hasCapability(newStream, durability))) { - throw new IllegalStateException("The procedure WAL relies on the ability to " + durability + + if (enforceStreamCapability && !(CommonFSUtils.hasCapability(newStream, durability))) { + throw new IllegalStateException("The procedure WAL relies on the ability to " + durability + " for proper operation during component failures, but the underlying filesystem does " + "not support doing so. Please check the config value of '" + USE_HSYNC_CONF_KEY + "' to set the desired level of robustness and ensure the config value of '" + diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml index d3501275a5..114ee8a23c 100644 --- a/hbase-procedure/src/test/resources/hbase-site.xml +++ b/hbase-procedure/src/test/resources/hbase-site.xml @@ -32,4 +32,13 @@ procedure to have an owner + + hbase.unsafe.stream.capability.enforce + false + + Controls whether HBase will check for stream capabilities (hflush/hsync). + Disable this if you intend to run on LocalFileSystem. + WARNING: Doing so may expose you to additional risk of data loss! + + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java index 6a7e4fa0b3..583759b66d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/AsyncFSOutputHelper.java @@ -73,8 +73,9 @@ public final class AsyncFSOutputHelper { // After we create the stream but before we attempt to use it at all // ensure that we can provide the level of data safety we're configured // to provide. - if (!(CommonFSUtils.hasCapability(fsOut, "hflush") && - CommonFSUtils.hasCapability(fsOut, "hsync"))) { + if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) && + !(CommonFSUtils.hasCapability(fsOut, "hflush") && + CommonFSUtils.hasCapability(fsOut, "hsync"))) { throw new CommonFSUtils.StreamLacksCapabilityException("hflush and hsync"); } final ExecutorService flushExecutor = diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java index 64acfba340..7a135c9fe2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java @@ -93,7 +93,8 @@ public class ProtobufLogWriter extends AbstractProtobufLogWriter this.output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, null); // TODO Be sure to add a check for hsync if this branch includes HBASE-19024 - if (!(CommonFSUtils.hasCapability(output, "hflush"))) { + if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true) && + !(CommonFSUtils.hasCapability(output, "hflush"))) { throw new StreamLacksCapabilityException("hflush"); } } diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 64a1964435..dbdf7765d7 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -158,4 +158,13 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.unsafe.stream.capability.enforce + false + + Controls whether HBase will check for stream capabilities (hflush/hsync). + Disable this if you intend to run on LocalFileSystem. + WARNING: Doing so may expose you to additional risk of data loss! + + -- 2.15.0