From 0b51cf90df709c615706aff4d4c2784ff7bb9a66 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Tue, 3 Jul 2018 16:50:22 +0800 Subject: [PATCH] HBASE-20839 Fallback to FSHLog if we can not instantiated AsyncFSWAL when user does not specify AsyncFSWAL explicitly --- .../hadoop/hbase/regionserver/wal/AsyncFSWAL.java | 2 +- .../org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java | 17 +++++++++++++++++ .../java/org/apache/hadoop/hbase/wal/WALFactory.java | 14 +++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java index 7f3e30b..b0c2549 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java index 56edb75..062b368 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java @@ -21,6 +21,9 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput; +import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper; +import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper; import org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL; import org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; @@ -115,4 +118,18 @@ public class AsyncFSWALProvider extends AbstractFSWALProvider { throw new IOException("cannot get log writer", e); } } + + /** + * Test whether we can load the helper classes for async dfs output. + */ + public static boolean load() { + try { + Class.forName(FanOutOneBlockAsyncDFSOutput.class.getName()); + Class.forName(FanOutOneBlockAsyncDFSOutputHelper.class.getName()); + Class.forName(FanOutOneBlockAsyncDFSOutputSaslHelper.class.getName()); + return true; + } catch (Throwable e) { + return false; + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java index 4f6a898..8eb1690 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java @@ -123,7 +123,19 @@ public class WALFactory { @VisibleForTesting public Class getProviderClass(String key, String defaultValue) { try { - return Providers.valueOf(conf.get(key, defaultValue)).clazz; + Providers provider = Providers.valueOf(conf.get(key, defaultValue)); + if (provider != Providers.defaultProvider) { + // User gives a wal provider explicitly, just use that one + return provider.clazz; + } + // AsyncFSWAL has better performance in most cases, and also uses less resources, we will try + // to use it if possible. But it deeply hacks into the internal of DFSClient so will be easily + // broken when upgrading hadoop. If it is broken, then we fall back to use FSHLog. + if (AsyncFSWALProvider.load()) { + return AsyncFSWALProvider.class; + } else { + return FSHLogProvider.class; + } } catch (IllegalArgumentException exception) { // Fall back to them specifying a class name // Note that the passed default class shouldn't actually be used, since the above only fails -- 2.7.4