From 6951fbbe8c611b795a623e4210df7f5c33207083 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 5 Sep 2018 15:26:16 -0700 Subject: [PATCH] HBASE-21155 Save on a few log strings and some churn in wal splitter by skipping out early if no logs in dir --- .../hadoop/hbase/master/SplitLogManager.java | 54 ++++++++++++---------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java index 2e2f8bf47d..1e4e2ce17e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java @@ -174,7 +174,7 @@ public class SplitLogManager { } FileStatus[] logfiles = FSUtils.listStatus(fs, logDir, filter); if (logfiles == null || logfiles.length == 0) { - LOG.info(logDir + " is empty dir, no logs to split"); + LOG.info(logDir + " dir is empty, no logs to split."); } else { Collections.addAll(fileStatus, logfiles); } @@ -235,29 +235,33 @@ public class SplitLogManager { PathFilter filter) throws IOException { MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " + logDirs + " for serverName=" + serverNames); - FileStatus[] logfiles = getFileList(logDirs, filter); - status.setStatus("Checking directory contents..."); - SplitLogCounters.tot_mgr_log_split_batch_start.increment(); - LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs + - " for " + serverNames); - long t = EnvironmentEdgeManager.currentTime(); long totalSize = 0; - TaskBatch batch = new TaskBatch(); - for (FileStatus lf : logfiles) { - // TODO If the log file is still being written to - which is most likely - // the case for the last log file - then its length will show up here - // as zero. The size of such a file can only be retrieved after - // recover-lease is done. totalSize will be under in most cases and the - // metrics that it drives will also be under-reported. - totalSize += lf.getLen(); - String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf); - if (!enqueueSplitTask(pathToLog, batch)) { - throw new IOException("duplicate log split scheduled for " + lf.getPath()); + TaskBatch batch = null; + long startTime = 0; + FileStatus[] logfiles = getFileList(logDirs, filter); + if (logfiles.length != 0) { + status.setStatus("Checking directory contents..."); + SplitLogCounters.tot_mgr_log_split_batch_start.increment(); + LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs + + " for " + serverNames); + startTime = EnvironmentEdgeManager.currentTime(); + batch = new TaskBatch(); + for (FileStatus lf : logfiles) { + // TODO If the log file is still being written to - which is most likely + // the case for the last log file - then its length will show up here + // as zero. The size of such a file can only be retrieved after + // recover-lease is done. totalSize will be under in most cases and the + // metrics that it drives will also be under-reported. + totalSize += lf.getLen(); + String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf); + if (!enqueueSplitTask(pathToLog, batch)) { + throw new IOException("duplicate log split scheduled for " + lf.getPath()); + } } + waitForSplittingCompletion(batch, status); } - waitForSplittingCompletion(batch, status); - if (batch.done != batch.installed) { + if (batch != null && batch.done != batch.installed) { batch.isDead = true; SplitLogCounters.tot_mgr_log_split_batch_err.increment(); LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed @@ -285,10 +289,10 @@ public class SplitLogManager { } SplitLogCounters.tot_mgr_log_split_batch_success.increment(); } - String msg = - "finished splitting (more than or equal to) " + totalSize + " bytes in " + batch.installed - + " log files in " + logDirs + " in " - + (EnvironmentEdgeManager.currentTime() - t) + "ms"; + String msg = "Finished splitting (more than or equal to) " + totalSize + + " bytes in " + ((batch == null)? 0: batch.installed) + + " log files in " + logDirs + " in " + + ((startTime == -1)? startTime: (EnvironmentEdgeManager.currentTime() - startTime)) + "ms"; status.markComplete(msg); LOG.info(msg); return totalSize; @@ -448,7 +452,7 @@ public class SplitLogManager { } deadWorkers.add(workerName); } - LOG.info("dead splitlog worker " + workerName); + LOG.info("Dead splitlog worker {}", workerName); } void handleDeadWorkers(Set serverNames) { -- 2.16.3