diff --git src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java index fb8020f..6a8252a 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java @@ -264,7 +264,7 @@ public class HLogSplitter { boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors", true); - long totalBytesToSplit = countTotalBytes(logfiles); + countTotalBytes(logfiles); splitSize = 0; outputSink.startWriterThreads(entryBuffers); @@ -396,7 +397,10 @@ public class HLogSplitter { status.markComplete("Failed: reporter.progress asked us to terminate"); return false; } + // Report progress every so many edits and/or files opened (opening a file + // takes a bit of time). int editsCount = 0; + int openedNewFile = 0; Entry entry; try { while ((entry = getNextLogLine(in,logPath, skipErrors)) != null) { @@ -408,9 +412,10 @@ public class HLogSplitter { WriterAndPath wap = (WriterAndPath)o; if (wap == null) { wap = createWAP(region, entry, rootDir, tmpname, fs, conf); + openedNewFile++; if (wap == null) { - // ignore edits from this region. It doesn't ezist anymore. - // It was probably already split. + // ignore edits from this region. It doesn't exist anymore. + // It was probably already split. logWriters.put(region, BAD_WRITER); continue; } else { @@ -419,13 +424,18 @@ public class HLogSplitter { } wap.w.append(entry); editsCount++; - if (editsCount % interval == 0) { - status.setStatus("Split " + editsCount + " edits"); + // If sufficient edits have passed OR we've opened a few files, check if + // we should report progress. + if (editsCount % interval == 0 || (openedNewFile > everyNopenedFiles)) { + // Zero out files counter each time we fall in here. + openedNewFile = 0; + String countsStr = "edits=" + editsCount + ", files=" + logWriters.size(); + status.setStatus("Split " + countsStr); long t1 = EnvironmentEdgeManager.currentTimeMillis(); if ((t1 - last_report_at) > period) { last_report_at = t; if (reporter != null && reporter.progress() == false) { - status.markComplete("Failed: reporter.progress asked us to terminate"); + status.markComplete("Failed: reporter.progress asked us to terminate; " + countsStr); progress_failed = true; return false; } @@ -476,10 +486,11 @@ public class HLogSplitter { } } } - String msg = ("processed " + editsCount + " edits across " + n + " regions" + - " threw away edits for " + (logWriters.size() - n) + " regions" + - " log file = " + logPath + - " is corrupted = " + isCorrupted); + String msg = "Processed " + editsCount + " edits across " + n + " regions" + + " threw away edits for " + (logWriters.size() - n) + " regions" + + "; log file=" + logPath + + ", corrupted=" + isCorrupted + + ", progress_failed=" + progress_failed; LOG.info(msg); status.markComplete(msg); }