From b14f812a18978eeb3d3c3d312b667640071c33dd Mon Sep 17 00:00:00 2001 From: Mike Grimes Date: Fri, 17 Nov 2017 15:41:27 -0800 Subject: [PATCH] HBASE-17165 Actually make use of retry in LoadIncrementHFiles tool --- .../hadoop/hbase/tool/LoadIncrementalHFiles.java | 23 +++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java index c457e224da..9f2e4cddea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java @@ -131,6 +131,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool { private final FsDelegationToken fsDelegationToken; private final UserProvider userProvider; private final int nrThreads; + private AtomicInteger numRetries; private final RpcControllerFactory rpcControllerFactory; private String bulkToken; @@ -177,6 +178,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool { maxFilesPerRegionPerFamily = conf.getInt(MAX_FILES_PER_REGION_PER_FAMILY, 32); nrThreads = conf.getInt("hbase.loadincremental.threads.max", Runtime.getRuntime().availableProcessors()); + numRetries = new AtomicInteger(1); rpcControllerFactory = new RpcControllerFactory(conf); } @@ -781,8 +783,8 @@ public class LoadIncrementalHFiles extends Configured implements Tool { protected List tryAtomicRegionLoad(ClientServiceCallable serviceCallable, final TableName tableName, final byte[] first, final Collection lqis) throws IOException { + List toRetry = new ArrayList<>(); try { - List toRetry = new ArrayList<>(); Configuration conf = getConf(); byte[] region = RpcRetryingCallerFactory.instantiate(conf, null). newCaller() .callWithRetries(serviceCallable, Integer.MAX_VALUE); @@ -796,8 +798,23 @@ public class LoadIncrementalHFiles extends Configured implements Tool { return toRetry; } catch (IOException e) { LOG.error("Encountered unrecoverable error from region server, additional details: " + - serviceCallable.getExceptionMessageAdditionalDetail(), - e); + serviceCallable.getExceptionMessageAdditionalDetail(), + e); + LOG.warn( + "Received a " + e.getClass().getSimpleName() + + " from region server: " + + serviceCallable.getExceptionMessageAdditionalDetail(), e); + if (getConf().getBoolean(RETRY_ON_IO_EXCEPTION, false) + && numRetries.get() < getConf().getInt( + HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER)) { + LOG.warn("Will attempt to retry loading failed HFiles. Retry #" + + numRetries.get()); + numRetries.getAndIncrement(); + toRetry.addAll(lqis); + return toRetry; + } + LOG.error(RETRY_ON_IO_EXCEPTION + " is disabled. Unable to recover"); throw e; } } -- 2.13.5 (Apple Git-94)