diff --git src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java index fb8020f..9abcf25 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java @@ -264,7 +264,7 @@ public class HLogSplitter { boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors", true); - long totalBytesToSplit = countTotalBytes(logfiles); + countTotalBytes(logfiles); splitSize = 0; outputSink.startWriterThreads(entryBuffers); @@ -369,10 +369,10 @@ public class HLogSplitter { boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors", HLog.SPLIT_SKIP_ERRORS_DEFAULT); int interval = conf.getInt("hbase.splitlog.report.interval.loglines", 1024); - // How often to send a progress report (default 1/2 master timeout) + // How often to send a progress report (default 1/2 the zookeeper session + // timeout of if that not set, the split log DEFAULT_TIMEOUT) int period = conf.getInt("hbase.splitlog.report.period", - conf.getInt("hbase.splitlog.manager.timeout", - ZKSplitLog.DEFAULT_TIMEOUT) / 2); + conf.getInt("zookeeper.session.timeout", ZKSplitLog.DEFAULT_TIMEOUT) / 2); Path logPath = logfile.getPath(); long logLength = logfile.getLen(); LOG.info("Splitting hlog: " + logPath + ", length=" + logLength); @@ -397,6 +397,9 @@ public class HLogSplitter { return false; } int editsCount = 0; + // Open of a new file takes some time so on each open, check if we should + // report progress. + boolean openedNewFile = false; Entry entry; try { while ((entry = getNextLogLine(in,logPath, skipErrors)) != null) { @@ -409,23 +412,28 @@ public class HLogSplitter { if (wap == null) { wap = createWAP(region, entry, rootDir, tmpname, fs, conf); if (wap == null) { - // ignore edits from this region. It doesn't ezist anymore. - // It was probably already split. + // ignore edits from this region. It doesn't ezist anymore. + // It was probably already split. logWriters.put(region, BAD_WRITER); continue; } else { logWriters.put(region, wap); } + openedNewFile = true; } wap.w.append(entry); editsCount++; - if (editsCount % interval == 0) { - status.setStatus("Split " + editsCount + " edits"); + // If sufficient edits have passed OR we've opened a file, check if + // we should report progress. + if (editsCount % interval == 0 || openedNewFile) { + openedNewFile = false; + String countsStr = "edits=" + editsCount + ", files=" + logWriters.size(); + status.setStatus("Split " + countsStr); long t1 = EnvironmentEdgeManager.currentTimeMillis(); if ((t1 - last_report_at) > period) { last_report_at = t; if (reporter != null && reporter.progress() == false) { - status.markComplete("Failed: reporter.progress asked us to terminate"); + status.markComplete("Failed: reporter.progress asked us to terminate; " + countsStr); progress_failed = true; return false; } @@ -476,10 +484,11 @@ public class HLogSplitter { } } } - String msg = ("processed " + editsCount + " edits across " + n + " regions" + - " threw away edits for " + (logWriters.size() - n) + " regions" + - " log file = " + logPath + - " is corrupted = " + isCorrupted); + String msg = "Processed " + editsCount + " edits across " + n + " regions" + + " threw away edits for " + (logWriters.size() - n) + " regions" + + "; log file=" + logPath + + ", corrupted=" + isCorrupted + + ", progress_failed=" + progress_failed; LOG.info(msg); status.markComplete(msg); }